onnxruntime 0.9.2-x86_64-darwin → 0.9.3-x86_64-darwin

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c2a0b2de5bfdb9b5b764990ba085ec0769aeee2785755953281c52709d667448
4
- data.tar.gz: f7e325c6bf12234a7903cadacd1e97dec47da63b92c85c7946bf5194ff6160c9
3
+ metadata.gz: 21a70412d3116efce9f5ee233b181ecd5b6746b824e7fcb8e18101351fda19be
4
+ data.tar.gz: 54e8e83d604487ca88a6641bca3aa1d1daefb767ba40c46d412334880e2de747
5
5
  SHA512:
6
- metadata.gz: f88582e2b05497ed40db53d2ccad9387e093ff100fd08c6f8bfaa7a6ac7d2d8c90393aada08618c24ea3ea76d8132f0d9d6a028013538fc6b0fac93936011c0d
7
- data.tar.gz: 8c843e981945bb9a1f097fd514d7a631ae7d1a1e5573eab45c86c78d71e429517363b0f71131f0a6a8c71c6df3915ee20f5a023bec3b5b6a764c14646b811e78
6
+ metadata.gz: fb35f77b9396c4d570ac33cee7a5cd252575f847bbd1c02c267f1bf11802de0016f78028a6ea01fe3069e21035be7fe9d390622e255523f5d8bb673e51c48a6d
7
+ data.tar.gz: c2aa31f7ee408220fccdd8405c493fcf34fb021e6b7c1d7826d9666cdcc8af90c71c9705aa8050f7ac5cc77d6dbf5730c85be2a109c29e409e52f02fa2ea1655
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 0.9.3 (2024-11-01)
2
+
3
+ - Updated ONNX Runtime to 1.20.0
4
+ - Added experimental `OrtValue` class
5
+ - Added experimental `run_with_ort_values` method
6
+
1
7
  ## 0.9.2 (2024-09-04)
2
8
 
3
9
  - Updated ONNX Runtime to 1.19.2
@@ -11,7 +11,7 @@ module OnnxRuntime
11
11
 
12
12
  # enums
13
13
  TensorElementDataType = enum(:undefined, :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :string, :bool, :float16, :double, :uint32, :uint64, :complex64, :complex128, :bfloat16)
14
- OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor)
14
+ OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor, :optional)
15
15
 
16
16
  class Api < ::FFI::Struct
17
17
  layout \
@@ -247,7 +247,11 @@ module OnnxRuntime
247
247
  attach_function :OrtGetApiBase, %i[], ApiBase.by_ref
248
248
 
249
249
  def self.api
250
- @api ||= self.OrtGetApiBase[:GetApi].call(ORT_API_VERSION)
250
+ @api ||= begin
251
+ api = self.OrtGetApiBase[:GetApi].call(ORT_API_VERSION)
252
+ api = Api.by_ref.from_native(api, nil) if RUBY_PLATFORM == "java"
253
+ api
254
+ end
251
255
  end
252
256
 
253
257
  if Gem.win_platform?
@@ -83,23 +83,36 @@ module OnnxRuntime
83
83
  @session = load_session(path_or_bytes, session_options)
84
84
  ObjectSpace.define_finalizer(@session, self.class.finalize(read_pointer.to_i))
85
85
 
86
- @allocator = load_allocator
86
+ @allocator = Utils.allocator
87
87
  @inputs = load_inputs
88
88
  @outputs = load_outputs
89
89
  ensure
90
90
  release :SessionOptions, session_options
91
91
  end
92
92
 
93
- # TODO support logid
94
93
  def run(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil, output_type: :ruby)
95
- # pointer references
96
- refs = []
94
+ if ![:ruby, :numo, :ort_value].include?(output_type)
95
+ raise ArgumentError, "Invalid output type: #{output_type}"
96
+ end
97
97
 
98
- input_tensor = create_input_tensor(input_feed, refs)
98
+ ort_values = input_feed.keys.zip(create_input_tensor(input_feed)).to_h
99
+
100
+ outputs = run_with_ort_values(output_names, ort_values, log_severity_level: log_severity_level, log_verbosity_level: log_verbosity_level, logid: logid, terminate: terminate)
101
+
102
+ outputs.map { |v| output_type == :numo ? v.numo : (output_type == :ort_value ? v : v.to_ruby) }
103
+ end
104
+
105
+ # TODO support logid
106
+ def run_with_ort_values(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil)
107
+ input_tensor = ::FFI::MemoryPointer.new(:pointer, input_feed.size)
108
+ input_feed.each_with_index do |(_, input), i|
109
+ input_tensor[i].write_pointer(input.to_ptr)
110
+ end
99
111
 
100
112
  output_names ||= @outputs.map { |v| v[:name] }
101
113
 
102
114
  output_tensor = ::FFI::MemoryPointer.new(:pointer, outputs.size)
115
+ refs = []
103
116
  input_node_names = create_node_names(input_feed.keys.map(&:to_s), refs)
104
117
  output_node_names = create_node_names(output_names.map(&:to_s), refs)
105
118
 
@@ -113,17 +126,9 @@ module OnnxRuntime
113
126
 
114
127
  check_status api[:Run].call(read_pointer, run_options.read_pointer, input_node_names, input_tensor, input_feed.size, output_node_names, output_names.size, output_tensor)
115
128
 
116
- output_names.size.times.map do |i|
117
- create_from_onnx_value(output_tensor[i].read_pointer, output_type)
118
- end
129
+ output_names.size.times.map { |i| OrtValue.new(output_tensor[i]) }
119
130
  ensure
120
131
  release :RunOptions, run_options
121
- if input_tensor
122
- input_feed.size.times do |i|
123
- release :Value, input_tensor[i]
124
- end
125
- end
126
- # output values released in create_from_onnx_value
127
132
  end
128
133
 
129
134
  def modelmeta
@@ -221,12 +226,6 @@ module OnnxRuntime
221
226
  session
222
227
  end
223
228
 
224
- def load_allocator
225
- allocator = ::FFI::MemoryPointer.new(:pointer)
226
- check_status api[:GetAllocatorWithDefaultOptions].call(allocator)
227
- allocator
228
- end
229
-
230
229
  def load_inputs
231
230
  inputs = []
232
231
  num_input_nodes = ::FFI::MemoryPointer.new(:size_t)
@@ -237,7 +236,7 @@ module OnnxRuntime
237
236
  # freed in node_info
238
237
  typeinfo = ::FFI::MemoryPointer.new(:pointer)
239
238
  check_status api[:SessionGetInputTypeInfo].call(read_pointer, i, typeinfo)
240
- inputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
239
+ inputs << {name: name_ptr.read_pointer.read_string}.merge(Utils.node_info(typeinfo))
241
240
  allocator_free name_ptr
242
241
  end
243
242
  inputs
@@ -253,91 +252,28 @@ module OnnxRuntime
253
252
  # freed in node_info
254
253
  typeinfo = ::FFI::MemoryPointer.new(:pointer)
255
254
  check_status api[:SessionGetOutputTypeInfo].call(read_pointer, i, typeinfo)
256
- outputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
255
+ outputs << {name: name_ptr.read_pointer.read_string}.merge(Utils.node_info(typeinfo))
257
256
  allocator_free name_ptr
258
257
  end
259
258
  outputs
260
259
  end
261
260
 
262
- def create_input_tensor(input_feed, refs)
263
- allocator_info = ::FFI::MemoryPointer.new(:pointer)
264
- check_status api[:CreateCpuMemoryInfo].call(1, 0, allocator_info)
265
- input_tensor = ::FFI::MemoryPointer.new(:pointer, input_feed.size)
266
-
267
- input_feed.each_with_index do |(input_name, input), idx|
261
+ def create_input_tensor(input_feed)
262
+ input_feed.map do |input_name, input|
268
263
  # TODO support more types
269
264
  inp = @inputs.find { |i| i[:name] == input_name.to_s }
270
265
  raise Error, "Unknown input: #{input_name}" unless inp
271
266
 
272
- input = input.to_a unless input.is_a?(Array) || numo_array?(input)
273
- shape = input_shape(input)
274
-
275
- input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
276
- input_node_dims.write_array_of_int64(shape)
277
-
278
- if inp[:type] == "tensor(string)"
279
- type_enum = FFI::TensorElementDataType[:string]
280
- check_status api[:CreateTensorAsOrtValue].call(@allocator.read_pointer, input_node_dims, shape.size, type_enum, input_tensor[idx])
281
-
282
- # keep reference to _str_ptrs until FillStringTensor call
283
- input_tensor_values, _str_ptrs = create_input_strings(input)
284
- check_status api[:FillStringTensor].call(input_tensor[idx].read_pointer, input_tensor_values, input_tensor_values.size / input_tensor_values.type_size)
267
+ if input.is_a?(OrtValue)
268
+ input
269
+ elsif inp[:type] == "tensor(string)"
270
+ OrtValue.from_array(input, element_type: :string)
285
271
  elsif (tensor_type = tensor_types[inp[:type]])
286
- input_tensor_values = create_input_data(input, tensor_type)
287
- type_enum = FFI::TensorElementDataType[tensor_type]
288
- check_status api[:CreateTensorWithDataAsOrtValue].call(allocator_info.read_pointer, input_tensor_values, input_tensor_values.size, input_node_dims, shape.size, type_enum, input_tensor[idx])
289
-
290
- refs << input_tensor_values
272
+ OrtValue.from_array(input, element_type: tensor_type)
291
273
  else
292
- unsupported_type("input", inp[:type])
274
+ Utils.unsupported_type("input", inp[:type])
293
275
  end
294
276
  end
295
-
296
- input_tensor
297
- ensure
298
- release :MemoryInfo, allocator_info
299
- end
300
-
301
- def input_shape(input)
302
- if numo_array?(input)
303
- input.shape
304
- else
305
- shape = []
306
- s = input
307
- while s.is_a?(Array)
308
- shape << s.size
309
- s = s.first
310
- end
311
- shape
312
- end
313
- end
314
-
315
- def create_input_strings(input)
316
- str_ptrs =
317
- if numo_array?(input)
318
- input.size.times.map { |i| ::FFI::MemoryPointer.from_string(input[i]) }
319
- else
320
- input.flatten.map { |v| ::FFI::MemoryPointer.from_string(v) }
321
- end
322
-
323
- input_tensor_values = ::FFI::MemoryPointer.new(:pointer, str_ptrs.size)
324
- input_tensor_values.write_array_of_pointer(str_ptrs)
325
- [input_tensor_values, str_ptrs]
326
- end
327
-
328
- def create_input_data(input, tensor_type)
329
- if numo_array?(input)
330
- input.cast_to(numo_types[tensor_type]).to_binary
331
- else
332
- flat_input = input.flatten.to_a
333
- input_tensor_values = ::FFI::MemoryPointer.new(tensor_type, flat_input.size)
334
- if tensor_type == :bool
335
- input_tensor_values.write_array_of_uint8(flat_input.map { |v| v ? 1 : 0 })
336
- else
337
- input_tensor_values.send("write_array_of_#{tensor_type}", flat_input)
338
- end
339
- input_tensor_values
340
- end
341
277
  end
342
278
 
343
279
  def create_node_names(names, refs)
@@ -349,230 +285,18 @@ module OnnxRuntime
349
285
  ptr
350
286
  end
351
287
 
352
- def create_from_onnx_value(out_ptr, output_type)
353
- out_type = ::FFI::MemoryPointer.new(:int)
354
- check_status api[:GetValueType].call(out_ptr, out_type)
355
- type = FFI::OnnxType[out_type.read_int]
356
-
357
- case type
358
- when :tensor
359
- typeinfo = ::FFI::MemoryPointer.new(:pointer)
360
- check_status api[:GetTensorTypeAndShape].call(out_ptr, typeinfo)
361
-
362
- type, shape = tensor_type_and_shape(typeinfo)
363
-
364
- tensor_data = ::FFI::MemoryPointer.new(:pointer)
365
- check_status api[:GetTensorMutableData].call(out_ptr, tensor_data)
366
-
367
- out_size = ::FFI::MemoryPointer.new(:size_t)
368
- check_status api[:GetTensorShapeElementCount].call(typeinfo.read_pointer, out_size)
369
- output_tensor_size = out_size.read(:size_t)
370
-
371
- release :TensorTypeAndShapeInfo, typeinfo
372
-
373
- # TODO support more types
374
- type = FFI::TensorElementDataType[type]
375
-
376
- case output_type
377
- when :numo
378
- case type
379
- when :string
380
- result = Numo::RObject.new(shape)
381
- result.allocate
382
- create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
383
- else
384
- numo_type = numo_types[type]
385
- unsupported_type("element", type) unless numo_type
386
- numo_type.from_binary(tensor_data.read_pointer.read_bytes(output_tensor_size * numo_type::ELEMENT_BYTE_SIZE), shape)
387
- end
388
- when :ruby
389
- arr =
390
- case type
391
- when :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :double, :uint32, :uint64
392
- tensor_data.read_pointer.send("read_array_of_#{type}", output_tensor_size)
393
- when :bool
394
- tensor_data.read_pointer.read_array_of_uint8(output_tensor_size).map { |v| v == 1 }
395
- when :string
396
- create_strings_from_onnx_value(out_ptr, output_tensor_size, [])
397
- else
398
- unsupported_type("element", type)
399
- end
400
-
401
- Utils.reshape(arr, shape)
402
- else
403
- raise ArgumentError, "Invalid output type: #{output_type}"
404
- end
405
- when :sequence
406
- out = ::FFI::MemoryPointer.new(:size_t)
407
- check_status api[:GetValueCount].call(out_ptr, out)
408
-
409
- out.read(:size_t).times.map do |i|
410
- seq = ::FFI::MemoryPointer.new(:pointer)
411
- check_status api[:GetValue].call(out_ptr, i, @allocator.read_pointer, seq)
412
- create_from_onnx_value(seq.read_pointer, output_type)
413
- end
414
- when :map
415
- type_shape = ::FFI::MemoryPointer.new(:pointer)
416
- map_keys = ::FFI::MemoryPointer.new(:pointer)
417
- map_values = ::FFI::MemoryPointer.new(:pointer)
418
- elem_type = ::FFI::MemoryPointer.new(:int)
419
-
420
- check_status api[:GetValue].call(out_ptr, 0, @allocator.read_pointer, map_keys)
421
- check_status api[:GetValue].call(out_ptr, 1, @allocator.read_pointer, map_values)
422
- check_status api[:GetTensorTypeAndShape].call(map_keys.read_pointer, type_shape)
423
- check_status api[:GetTensorElementType].call(type_shape.read_pointer, elem_type)
424
- release :TensorTypeAndShapeInfo, type_shape
425
-
426
- # TODO support more types
427
- elem_type = FFI::TensorElementDataType[elem_type.read_int]
428
- case elem_type
429
- when :int64
430
- ret = {}
431
- keys = create_from_onnx_value(map_keys.read_pointer, output_type)
432
- values = create_from_onnx_value(map_values.read_pointer, output_type)
433
- keys.zip(values).each do |k, v|
434
- ret[k] = v
435
- end
436
- ret
437
- else
438
- unsupported_type("element", elem_type)
439
- end
440
- else
441
- unsupported_type("ONNX", type)
442
- end
443
- ensure
444
- api[:ReleaseValue].call(out_ptr) unless out_ptr.null?
445
- end
446
-
447
- def create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
448
- len = ::FFI::MemoryPointer.new(:size_t)
449
- check_status api[:GetStringTensorDataLength].call(out_ptr, len)
450
-
451
- s_len = len.read(:size_t)
452
- s = ::FFI::MemoryPointer.new(:uchar, s_len)
453
- offsets = ::FFI::MemoryPointer.new(:size_t, output_tensor_size)
454
- check_status api[:GetStringTensorContent].call(out_ptr, s, s_len, offsets, output_tensor_size)
455
-
456
- offsets = output_tensor_size.times.map { |i| offsets[i].read(:size_t) }
457
- offsets << s_len
458
- output_tensor_size.times do |i|
459
- result[i] = s.get_bytes(offsets[i], offsets[i + 1] - offsets[i])
460
- end
461
- result
462
- end
463
-
464
288
  def read_pointer
465
289
  @session.read_pointer
466
290
  end
467
291
 
468
292
  def check_status(status)
469
- unless status.null?
470
- message = api[:GetErrorMessage].call(status).read_string
471
- api[:ReleaseStatus].call(status)
472
- raise Error, message
473
- end
474
- end
475
-
476
- def node_info(typeinfo)
477
- onnx_type = ::FFI::MemoryPointer.new(:int)
478
- check_status api[:GetOnnxTypeFromTypeInfo].call(typeinfo.read_pointer, onnx_type)
479
-
480
- type = FFI::OnnxType[onnx_type.read_int]
481
- case type
482
- when :tensor
483
- tensor_info = ::FFI::MemoryPointer.new(:pointer)
484
- # don't free tensor_info
485
- check_status api[:CastTypeInfoToTensorInfo].call(typeinfo.read_pointer, tensor_info)
486
-
487
- type, shape = tensor_type_and_shape(tensor_info)
488
- {
489
- type: "tensor(#{FFI::TensorElementDataType[type]})",
490
- shape: shape
491
- }
492
- when :sequence
493
- sequence_type_info = ::FFI::MemoryPointer.new(:pointer)
494
- check_status api[:CastTypeInfoToSequenceTypeInfo].call(typeinfo.read_pointer, sequence_type_info)
495
- nested_type_info = ::FFI::MemoryPointer.new(:pointer)
496
- check_status api[:GetSequenceElementType].call(sequence_type_info.read_pointer, nested_type_info)
497
- v = node_info(nested_type_info)[:type]
498
-
499
- {
500
- type: "seq(#{v})",
501
- shape: []
502
- }
503
- when :map
504
- map_type_info = ::FFI::MemoryPointer.new(:pointer)
505
- check_status api[:CastTypeInfoToMapTypeInfo].call(typeinfo.read_pointer, map_type_info)
506
-
507
- # key
508
- key_type = ::FFI::MemoryPointer.new(:int)
509
- check_status api[:GetMapKeyType].call(map_type_info.read_pointer, key_type)
510
- k = FFI::TensorElementDataType[key_type.read_int]
511
-
512
- # value
513
- value_type_info = ::FFI::MemoryPointer.new(:pointer)
514
- check_status api[:GetMapValueType].call(map_type_info.read_pointer, value_type_info)
515
- v = node_info(value_type_info)[:type]
516
-
517
- {
518
- type: "map(#{k},#{v})",
519
- shape: []
520
- }
521
- else
522
- unsupported_type("ONNX", type)
523
- end
524
- ensure
525
- release :TypeInfo, typeinfo
526
- end
527
-
528
- def tensor_type_and_shape(tensor_info)
529
- type = ::FFI::MemoryPointer.new(:int)
530
- check_status api[:GetTensorElementType].call(tensor_info.read_pointer, type)
531
-
532
- num_dims_ptr = ::FFI::MemoryPointer.new(:size_t)
533
- check_status api[:GetDimensionsCount].call(tensor_info.read_pointer, num_dims_ptr)
534
- num_dims = num_dims_ptr.read(:size_t)
535
-
536
- node_dims = ::FFI::MemoryPointer.new(:int64, num_dims)
537
- check_status api[:GetDimensions].call(tensor_info.read_pointer, node_dims, num_dims)
538
- dims = node_dims.read_array_of_int64(num_dims)
539
-
540
- symbolic_dims = ::FFI::MemoryPointer.new(:pointer, num_dims)
541
- check_status api[:GetSymbolicDimensions].call(tensor_info.read_pointer, symbolic_dims, num_dims)
542
- named_dims = num_dims.times.map { |i| symbolic_dims[i].read_pointer.read_string }
543
- dims = named_dims.zip(dims).map { |n, d| n.empty? ? d : n }
544
-
545
- [type.read_int, dims]
546
- end
547
-
548
- def unsupported_type(name, type)
549
- raise Error, "Unsupported #{name} type: #{type}"
293
+ Utils.check_status(status)
550
294
  end
551
295
 
552
296
  def tensor_types
553
297
  @tensor_types ||= [:float, :uint8, :int8, :uint16, :int16, :int32, :int64, :bool, :double, :uint32, :uint64].map { |v| ["tensor(#{v})", v] }.to_h
554
298
  end
555
299
 
556
- def numo_array?(obj)
557
- defined?(Numo::NArray) && obj.is_a?(Numo::NArray)
558
- end
559
-
560
- def numo_types
561
- @numo_types ||= {
562
- float: Numo::SFloat,
563
- uint8: Numo::UInt8,
564
- int8: Numo::Int8,
565
- uint16: Numo::UInt16,
566
- int16: Numo::Int16,
567
- int32: Numo::Int32,
568
- int64: Numo::Int64,
569
- bool: Numo::UInt8,
570
- double: Numo::DFloat,
571
- uint32: Numo::UInt32,
572
- uint64: Numo::UInt64
573
- }
574
- end
575
-
576
300
  def api
577
301
  self.class.api
578
302
  end
@@ -590,7 +314,7 @@ module OnnxRuntime
590
314
  end
591
315
 
592
316
  def self.release(type, pointer)
593
- api[:"Release#{type}"].call(pointer.read_pointer) if pointer && !pointer.null?
317
+ Utils.release(type, pointer)
594
318
  end
595
319
 
596
320
  def self.finalize(addr)
@@ -0,0 +1,278 @@
1
+ module OnnxRuntime
2
+ class OrtValue
3
+ def initialize(ptr, ref = nil)
4
+ @ptr = ptr.read_pointer
5
+ @ref = ref # keep reference to data
6
+ ObjectSpace.define_finalizer(@ptr, self.class.finalize(@ptr.to_i))
7
+ end
8
+
9
+ def self.from_numo(numo_obj)
10
+ element_type = numo_obj.is_a?(Numo::Bit) ? :bool : Utils.numo_types.invert[numo_obj.class]
11
+ Utils.unsupported_type("Numo", numo_obj.class.name) unless element_type
12
+
13
+ from_array(numo_obj, element_type: element_type)
14
+ end
15
+
16
+ def self.from_array(input, element_type:)
17
+ type_enum = FFI::TensorElementDataType[element_type]
18
+ Utils.unsupported_type("element", element_type) unless type_enum
19
+
20
+ input = input.to_a unless input.is_a?(Array) || Utils.numo_array?(input)
21
+
22
+ shape = Utils.input_shape(input)
23
+ input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
24
+ input_node_dims.write_array_of_int64(shape)
25
+
26
+ ptr = ::FFI::MemoryPointer.new(:pointer)
27
+ if element_type == :string
28
+ # keep reference to _str_ptrs until FillStringTensor call
29
+ input_tensor_values, _str_ptrs = create_input_strings(input)
30
+ Utils.check_status FFI.api[:CreateTensorAsOrtValue].call(Utils.allocator.read_pointer, input_node_dims, shape.size, type_enum, ptr)
31
+ Utils.check_status FFI.api[:FillStringTensor].call(ptr.read_pointer, input_tensor_values, input_tensor_values.size / input_tensor_values.type_size)
32
+ else
33
+ input_tensor_values = create_input_data(input, element_type)
34
+ Utils.check_status FFI.api[:CreateTensorWithDataAsOrtValue].call(allocator_info.read_pointer, input_tensor_values, input_tensor_values.size, input_node_dims, shape.size, type_enum, ptr)
35
+ end
36
+
37
+ new(ptr, input_tensor_values)
38
+ end
39
+
40
+ def self.from_shape_and_type(shape, element_type)
41
+ type_enum = FFI::TensorElementDataType[element_type]
42
+ Utils.unsupported_type("element", element_type) unless type_enum
43
+
44
+ input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
45
+ input_node_dims.write_array_of_int64(shape)
46
+
47
+ ptr = ::FFI::MemoryPointer.new(:pointer)
48
+ Utils.check_status FFI.api[:CreateTensorAsOrtValue].call(Utils.allocator.read_pointer, input_node_dims, shape.size, type_enum, ptr)
49
+
50
+ new(ptr)
51
+ end
52
+
53
+ def self.create_input_data(input, tensor_type)
54
+ if Utils.numo_array?(input)
55
+ input.cast_to(Utils.numo_types[tensor_type]).to_binary
56
+ else
57
+ flat_input = input.flatten.to_a
58
+ input_tensor_values = ::FFI::MemoryPointer.new(tensor_type, flat_input.size)
59
+ if tensor_type == :bool
60
+ input_tensor_values.write_array_of_uint8(flat_input.map { |v| v ? 1 : 0 })
61
+ else
62
+ input_tensor_values.send("write_array_of_#{tensor_type}", flat_input)
63
+ end
64
+ input_tensor_values
65
+ end
66
+ end
67
+ private_class_method :create_input_data
68
+
69
+ def self.create_input_strings(input)
70
+ str_ptrs =
71
+ if Utils.numo_array?(input)
72
+ input.size.times.map { |i| ::FFI::MemoryPointer.from_string(input[i]) }
73
+ else
74
+ input.flatten.map { |v| ::FFI::MemoryPointer.from_string(v) }
75
+ end
76
+
77
+ input_tensor_values = ::FFI::MemoryPointer.new(:pointer, str_ptrs.size)
78
+ input_tensor_values.write_array_of_pointer(str_ptrs)
79
+ [input_tensor_values, str_ptrs]
80
+ end
81
+ private_class_method :create_input_strings
82
+
83
+ def tensor?
84
+ FFI::OnnxType[value_type] == :tensor
85
+ end
86
+
87
+ def data_type
88
+ @data_type ||= begin
89
+ typeinfo = ::FFI::MemoryPointer.new(:pointer)
90
+ Utils.check_status FFI.api[:GetTypeInfo].call(@ptr, typeinfo)
91
+ Utils.node_info(typeinfo)[:type]
92
+ end
93
+ end
94
+
95
+ def element_type
96
+ FFI::TensorElementDataType[type_and_shape_info[0]]
97
+ end
98
+
99
+ def shape
100
+ type_and_shape_info[1]
101
+ end
102
+
103
+ def device_name
104
+ "cpu"
105
+ end
106
+
107
+ def numo
108
+ create_from_onnx_value(@ptr, :numo)
109
+ end
110
+
111
+ def to_ruby
112
+ create_from_onnx_value(@ptr, :ruby)
113
+ end
114
+
115
+ def to_ptr
116
+ @ptr
117
+ end
118
+
119
+ def data_ptr
120
+ tensor_data = ::FFI::MemoryPointer.new(:pointer)
121
+ FFI.api[:GetTensorMutableData].call(@ptr, tensor_data)
122
+ tensor_data.read_pointer
123
+ end
124
+
125
+ private
126
+
127
+ def value_type
128
+ @value_type ||= begin
129
+ out_type = ::FFI::MemoryPointer.new(:int)
130
+ Utils.check_status FFI.api[:GetValueType].call(@ptr, out_type)
131
+ out_type.read_int
132
+ end
133
+ end
134
+
135
+ def type_and_shape_info
136
+ @type_and_shape_info ||= begin
137
+ begin
138
+ typeinfo = ::FFI::MemoryPointer.new(:pointer)
139
+ Utils.check_status FFI.api[:GetTensorTypeAndShape].call(@ptr, typeinfo)
140
+ Utils.tensor_type_and_shape(typeinfo)
141
+ ensure
142
+ Utils.release :TensorTypeAndShapeInfo, typeinfo
143
+ end
144
+ end
145
+ end
146
+
147
+ def create_from_onnx_value(out_ptr, output_type)
148
+ out_type = ::FFI::MemoryPointer.new(:int)
149
+ Utils.check_status FFI.api[:GetValueType].call(out_ptr, out_type)
150
+ type = FFI::OnnxType[out_type.read_int]
151
+
152
+ case type
153
+ when :tensor
154
+ typeinfo = ::FFI::MemoryPointer.new(:pointer)
155
+ Utils.check_status FFI.api[:GetTensorTypeAndShape].call(out_ptr, typeinfo)
156
+
157
+ type, shape = Utils.tensor_type_and_shape(typeinfo)
158
+
159
+ tensor_data = ::FFI::MemoryPointer.new(:pointer)
160
+ Utils.check_status FFI.api[:GetTensorMutableData].call(out_ptr, tensor_data)
161
+
162
+ out_size = ::FFI::MemoryPointer.new(:size_t)
163
+ Utils.check_status FFI.api[:GetTensorShapeElementCount].call(typeinfo.read_pointer, out_size)
164
+ output_tensor_size = out_size.read(:size_t)
165
+
166
+ Utils.release :TensorTypeAndShapeInfo, typeinfo
167
+
168
+ # TODO support more types
169
+ type = FFI::TensorElementDataType[type]
170
+
171
+ case output_type
172
+ when :numo
173
+ case type
174
+ when :string
175
+ result = Numo::RObject.new(shape)
176
+ result.allocate
177
+ create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
178
+ else
179
+ numo_type = Utils.numo_types[type]
180
+ Utils.unsupported_type("element", type) unless numo_type
181
+ numo_type.from_binary(tensor_data.read_pointer.read_bytes(output_tensor_size * numo_type::ELEMENT_BYTE_SIZE), shape)
182
+ end
183
+ when :ruby
184
+ arr =
185
+ case type
186
+ when :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :double, :uint32, :uint64
187
+ tensor_data.read_pointer.send("read_array_of_#{type}", output_tensor_size)
188
+ when :bool
189
+ tensor_data.read_pointer.read_array_of_uint8(output_tensor_size).map { |v| v == 1 }
190
+ when :string
191
+ create_strings_from_onnx_value(out_ptr, output_tensor_size, [])
192
+ else
193
+ Utils.unsupported_type("element", type)
194
+ end
195
+
196
+ reshape(arr, shape)
197
+ else
198
+ raise ArgumentError, "Invalid output type: #{output_type}"
199
+ end
200
+ when :sequence
201
+ out = ::FFI::MemoryPointer.new(:size_t)
202
+ Utils.check_status FFI.api[:GetValueCount].call(out_ptr, out)
203
+
204
+ out.read(:size_t).times.map do |i|
205
+ seq = ::FFI::MemoryPointer.new(:pointer)
206
+ Utils.check_status FFI.api[:GetValue].call(out_ptr, i, Utils.allocator.read_pointer, seq)
207
+ create_from_onnx_value(seq.read_pointer, output_type)
208
+ end
209
+ when :map
210
+ type_shape = ::FFI::MemoryPointer.new(:pointer)
211
+ map_keys = ::FFI::MemoryPointer.new(:pointer)
212
+ map_values = ::FFI::MemoryPointer.new(:pointer)
213
+ elem_type = ::FFI::MemoryPointer.new(:int)
214
+
215
+ Utils.check_status FFI.api[:GetValue].call(out_ptr, 0, Utils.allocator.read_pointer, map_keys)
216
+ Utils.check_status FFI.api[:GetValue].call(out_ptr, 1, Utils.allocator.read_pointer, map_values)
217
+ Utils.check_status FFI.api[:GetTensorTypeAndShape].call(map_keys.read_pointer, type_shape)
218
+ Utils.check_status FFI.api[:GetTensorElementType].call(type_shape.read_pointer, elem_type)
219
+ Utils.release :TensorTypeAndShapeInfo, type_shape
220
+
221
+ # TODO support more types
222
+ elem_type = FFI::TensorElementDataType[elem_type.read_int]
223
+ case elem_type
224
+ when :int64
225
+ ret = {}
226
+ keys = create_from_onnx_value(map_keys.read_pointer, output_type)
227
+ values = create_from_onnx_value(map_values.read_pointer, output_type)
228
+ keys.zip(values).each do |k, v|
229
+ ret[k] = v
230
+ end
231
+ ret
232
+ else
233
+ Utils.unsupported_type("element", elem_type)
234
+ end
235
+ else
236
+ Utils.unsupported_type("ONNX", type)
237
+ end
238
+ end
239
+
240
+ def create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
241
+ len = ::FFI::MemoryPointer.new(:size_t)
242
+ Utils.check_status FFI.api[:GetStringTensorDataLength].call(out_ptr, len)
243
+
244
+ s_len = len.read(:size_t)
245
+ s = ::FFI::MemoryPointer.new(:uchar, s_len)
246
+ offsets = ::FFI::MemoryPointer.new(:size_t, output_tensor_size)
247
+ Utils.check_status FFI.api[:GetStringTensorContent].call(out_ptr, s, s_len, offsets, output_tensor_size)
248
+
249
+ offsets = output_tensor_size.times.map { |i| offsets[i].read(:size_t) }
250
+ offsets << s_len
251
+ output_tensor_size.times do |i|
252
+ result[i] = s.get_bytes(offsets[i], offsets[i + 1] - offsets[i])
253
+ end
254
+ result
255
+ end
256
+
257
+ def reshape(arr, dims)
258
+ arr = arr.flatten
259
+ dims[1..-1].reverse_each do |dim|
260
+ arr = arr.each_slice(dim)
261
+ end
262
+ arr.to_a
263
+ end
264
+
265
+ def self.finalize(addr)
266
+ # must use proc instead of stabby lambda
267
+ proc { FFI.api[:ReleaseValue].call(::FFI::Pointer.new(:pointer, addr)) }
268
+ end
269
+
270
+ def self.allocator_info
271
+ @allocator_info ||= begin
272
+ allocator_info = ::FFI::MemoryPointer.new(:pointer)
273
+ Utils.check_status FFI.api[:CreateCpuMemoryInfo].call(1, 0, allocator_info)
274
+ allocator_info
275
+ end
276
+ end
277
+ end
278
+ end
@@ -5,12 +5,138 @@ module OnnxRuntime
5
5
  end
6
6
  self.mutex = Mutex.new
7
7
 
8
- def self.reshape(arr, dims)
9
- arr = arr.flatten
10
- dims[1..-1].reverse.each do |dim|
11
- arr = arr.each_slice(dim)
8
+ def self.check_status(status)
9
+ unless status.null?
10
+ message = api[:GetErrorMessage].call(status).read_string
11
+ api[:ReleaseStatus].call(status)
12
+ raise Error, message
13
+ end
14
+ end
15
+
16
+ def self.api
17
+ FFI.api
18
+ end
19
+
20
+ def self.release(type, pointer)
21
+ FFI.api[:"Release#{type}"].call(pointer.read_pointer) if pointer && !pointer.null?
22
+ end
23
+
24
+ def self.unsupported_type(name, type)
25
+ raise Error, "Unsupported #{name} type: #{type}"
26
+ end
27
+
28
+ def self.tensor_type_and_shape(tensor_info)
29
+ type = ::FFI::MemoryPointer.new(:int)
30
+ check_status api[:GetTensorElementType].call(tensor_info.read_pointer, type)
31
+
32
+ num_dims_ptr = ::FFI::MemoryPointer.new(:size_t)
33
+ check_status api[:GetDimensionsCount].call(tensor_info.read_pointer, num_dims_ptr)
34
+ num_dims = num_dims_ptr.read(:size_t)
35
+
36
+ node_dims = ::FFI::MemoryPointer.new(:int64, num_dims)
37
+ check_status api[:GetDimensions].call(tensor_info.read_pointer, node_dims, num_dims)
38
+ dims = node_dims.read_array_of_int64(num_dims)
39
+
40
+ symbolic_dims = ::FFI::MemoryPointer.new(:pointer, num_dims)
41
+ check_status api[:GetSymbolicDimensions].call(tensor_info.read_pointer, symbolic_dims, num_dims)
42
+ named_dims = num_dims.times.map { |i| symbolic_dims[i].read_pointer.read_string }
43
+ dims = named_dims.zip(dims).map { |n, d| n.empty? ? d : n }
44
+
45
+ [type.read_int, dims]
46
+ end
47
+
48
+ def self.node_info(typeinfo)
49
+ onnx_type = ::FFI::MemoryPointer.new(:int)
50
+ check_status api[:GetOnnxTypeFromTypeInfo].call(typeinfo.read_pointer, onnx_type)
51
+
52
+ type = FFI::OnnxType[onnx_type.read_int]
53
+ case type
54
+ when :tensor
55
+ tensor_info = ::FFI::MemoryPointer.new(:pointer)
56
+ # don't free tensor_info
57
+ check_status api[:CastTypeInfoToTensorInfo].call(typeinfo.read_pointer, tensor_info)
58
+
59
+ type, shape = Utils.tensor_type_and_shape(tensor_info)
60
+ {
61
+ type: "tensor(#{FFI::TensorElementDataType[type]})",
62
+ shape: shape
63
+ }
64
+ when :sequence
65
+ sequence_type_info = ::FFI::MemoryPointer.new(:pointer)
66
+ check_status api[:CastTypeInfoToSequenceTypeInfo].call(typeinfo.read_pointer, sequence_type_info)
67
+ nested_type_info = ::FFI::MemoryPointer.new(:pointer)
68
+ check_status api[:GetSequenceElementType].call(sequence_type_info.read_pointer, nested_type_info)
69
+ v = node_info(nested_type_info)[:type]
70
+
71
+ {
72
+ type: "seq(#{v})",
73
+ shape: []
74
+ }
75
+ when :map
76
+ map_type_info = ::FFI::MemoryPointer.new(:pointer)
77
+ check_status api[:CastTypeInfoToMapTypeInfo].call(typeinfo.read_pointer, map_type_info)
78
+
79
+ # key
80
+ key_type = ::FFI::MemoryPointer.new(:int)
81
+ check_status api[:GetMapKeyType].call(map_type_info.read_pointer, key_type)
82
+ k = FFI::TensorElementDataType[key_type.read_int]
83
+
84
+ # value
85
+ value_type_info = ::FFI::MemoryPointer.new(:pointer)
86
+ check_status api[:GetMapValueType].call(map_type_info.read_pointer, value_type_info)
87
+ v = node_info(value_type_info)[:type]
88
+
89
+ {
90
+ type: "map(#{k},#{v})",
91
+ shape: []
92
+ }
93
+ else
94
+ Utils.unsupported_type("ONNX", type)
95
+ end
96
+ ensure
97
+ release :TypeInfo, typeinfo
98
+ end
99
+
100
+ def self.numo_array?(obj)
101
+ defined?(Numo::NArray) && obj.is_a?(Numo::NArray)
102
+ end
103
+
104
+ def self.numo_types
105
+ @numo_types ||= {
106
+ float: Numo::SFloat,
107
+ uint8: Numo::UInt8,
108
+ int8: Numo::Int8,
109
+ uint16: Numo::UInt16,
110
+ int16: Numo::Int16,
111
+ int32: Numo::Int32,
112
+ int64: Numo::Int64,
113
+ bool: Numo::UInt8,
114
+ double: Numo::DFloat,
115
+ uint32: Numo::UInt32,
116
+ uint64: Numo::UInt64
117
+ }
118
+ end
119
+
120
+ def self.input_shape(input)
121
+ if numo_array?(input)
122
+ input.shape
123
+ else
124
+ shape = []
125
+ s = input
126
+ while s.is_a?(Array)
127
+ shape << s.size
128
+ s = s.first
129
+ end
130
+ shape
131
+ end
132
+ end
133
+
134
+ def self.allocator
135
+ @allocator ||= begin
136
+ allocator = ::FFI::MemoryPointer.new(:pointer)
137
+ check_status api[:GetAllocatorWithDefaultOptions].call(allocator)
138
+ allocator
12
139
  end
13
- arr.to_a
14
140
  end
15
141
  end
16
142
  end
@@ -1,3 +1,3 @@
1
1
  module OnnxRuntime
2
- VERSION = "0.9.2"
2
+ VERSION = "0.9.3"
3
3
  end
data/lib/onnxruntime.rb CHANGED
@@ -5,6 +5,7 @@ require "ffi"
5
5
  require_relative "onnxruntime/datasets"
6
6
  require_relative "onnxruntime/inference_session"
7
7
  require_relative "onnxruntime/model"
8
+ require_relative "onnxruntime/ort_value"
8
9
  require_relative "onnxruntime/utils"
9
10
  require_relative "onnxruntime/version"
10
11
 
Binary file
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: onnxruntime
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.2
4
+ version: 0.9.3
5
5
  platform: x86_64-darwin
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-09-04 00:00:00.000000000 Z
11
+ date: 2024-11-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ffi
@@ -38,6 +38,7 @@ files:
38
38
  - lib/onnxruntime/ffi.rb
39
39
  - lib/onnxruntime/inference_session.rb
40
40
  - lib/onnxruntime/model.rb
41
+ - lib/onnxruntime/ort_value.rb
41
42
  - lib/onnxruntime/utils.rb
42
43
  - lib/onnxruntime/version.rb
43
44
  - vendor/LICENSE
@@ -62,7 +63,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
62
63
  - !ruby/object:Gem::Version
63
64
  version: '0'
64
65
  requirements: []
65
- rubygems_version: 3.5.11
66
+ rubygems_version: 3.5.16
66
67
  signing_key:
67
68
  specification_version: 4
68
69
  summary: High performance scoring engine for ML models