tensor_stream 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +5 -5
  2. data/.rubocop.yml +6 -1
  3. data/CHANGELOG.md +10 -0
  4. data/README.md +35 -0
  5. data/lib/tensor_stream.rb +2 -2
  6. data/lib/tensor_stream/debugging/debugging.rb +2 -1
  7. data/lib/tensor_stream/dynamic_stitch.rb +23 -24
  8. data/lib/tensor_stream/evaluator/base_evaluator.rb +27 -18
  9. data/lib/tensor_stream/evaluator/opencl/kernels/apply_momentum.cl +16 -0
  10. data/lib/tensor_stream/evaluator/opencl/kernels/pack.cl +24 -0
  11. data/lib/tensor_stream/evaluator/opencl/kernels/softmax_cross.cl +6 -1
  12. data/lib/tensor_stream/evaluator/opencl/opencl_buffer.rb +6 -6
  13. data/lib/tensor_stream/evaluator/opencl/opencl_evaluator.rb +237 -107
  14. data/lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb +97 -7
  15. data/lib/tensor_stream/evaluator/ruby_evaluator.rb +230 -123
  16. data/lib/tensor_stream/exceptions.rb +1 -0
  17. data/lib/tensor_stream/graph_builder.rb +2 -3
  18. data/lib/tensor_stream/graph_deserializers/protobuf.rb +22 -23
  19. data/lib/tensor_stream/graph_serializers/graphml.rb +26 -29
  20. data/lib/tensor_stream/graph_serializers/pbtext.rb +22 -19
  21. data/lib/tensor_stream/helpers/string_helper.rb +4 -5
  22. data/lib/tensor_stream/math_gradients.rb +141 -77
  23. data/lib/tensor_stream/nn/nn_ops.rb +4 -6
  24. data/lib/tensor_stream/operation.rb +139 -120
  25. data/lib/tensor_stream/ops.rb +36 -3
  26. data/lib/tensor_stream/session.rb +7 -11
  27. data/lib/tensor_stream/tensor.rb +3 -3
  28. data/lib/tensor_stream/tensor_shape.rb +5 -0
  29. data/lib/tensor_stream/train/gradient_descent_optimizer.rb +4 -37
  30. data/lib/tensor_stream/train/momentum_optimizer.rb +48 -0
  31. data/lib/tensor_stream/train/optimizer.rb +129 -0
  32. data/lib/tensor_stream/train/saver.rb +0 -1
  33. data/lib/tensor_stream/train/slot_creator.rb +62 -0
  34. data/lib/tensor_stream/train/utils.rb +11 -12
  35. data/lib/tensor_stream/trainer.rb +3 -0
  36. data/lib/tensor_stream/utils.rb +18 -11
  37. data/lib/tensor_stream/variable.rb +19 -12
  38. data/lib/tensor_stream/variable_scope.rb +1 -1
  39. data/lib/tensor_stream/version.rb +1 -1
  40. data/samples/iris.rb +2 -1
  41. data/samples/linear_regression.rb +3 -1
  42. data/samples/nearest_neighbor.rb +2 -0
  43. data/test_samples/neural_network_raw.py +101 -0
  44. data/test_samples/raw_neural_net_sample.rb +6 -4
  45. data/test_samples/test2.py +73 -27
  46. metadata +9 -3
@@ -6,11 +6,11 @@ module TensorStream
6
6
  start_index = start.shift
7
7
  dimen_size = start_index + size.shift
8
8
 
9
- input[start_index...dimen_size].collect do |input|
10
- if input.is_a?(Array)
11
- slice_tensor(input, start.dup, size.dup)
9
+ input[start_index...dimen_size].collect do |item|
10
+ if item.is_a?(Array)
11
+ slice_tensor(item, start.dup, size.dup)
12
12
  else
13
- input
13
+ item
14
14
  end
15
15
  end
16
16
  end
@@ -25,7 +25,7 @@ module TensorStream
25
25
 
26
26
  def _reduced_shape(input_shape, axes)
27
27
  return [] if axes.nil? # reduce to scalar
28
- axes = [ axes ] unless axes.is_a?(Array)
28
+ axes = [axes] unless axes.is_a?(Array)
29
29
  return input_shape if axes.empty?
30
30
 
31
31
  axes.each do |dimen|
@@ -72,8 +72,8 @@ module TensorStream
72
72
  d = dims.shift
73
73
 
74
74
  if input.is_a?(Array) && (get_rank(input) - 1) == dims.size
75
- row_to_dup = input.collect do |input|
76
- broadcast_dimensions(input, dims.dup)
75
+ row_to_dup = input.collect do |item|
76
+ broadcast_dimensions(item, dims.dup)
77
77
  end
78
78
 
79
79
  row_to_dup + Array.new(d) { row_to_dup }.flatten(1)
@@ -205,5 +205,95 @@ module TensorStream
205
205
  end
206
206
  end
207
207
  end
208
+
209
+ def gather(params, indexes)
210
+ indexes.collect do |index|
211
+ if index.is_a?(Array)
212
+ gather(params, index)
213
+ else
214
+ params[index]
215
+ end
216
+ end
217
+ end
218
+
219
+ # general case transposition with flat arrays
220
+ def transpose_with_perm(arr, new_arr, shape, new_shape, perm)
221
+ arr_size = shape.reduce(:*)
222
+ divisors = shape.dup.drop(1).reverse.inject([1]) do |a, s|
223
+ a << s * a.last
224
+ end.reverse
225
+
226
+ multipliers = new_shape.dup.drop(1).reverse.inject([1]) do |a, s|
227
+ a << s * a.last
228
+ end.reverse
229
+
230
+ arr_size.times do |p|
231
+ ptr = p
232
+ index = []
233
+ divisors.each_with_object(index) do |div, a|
234
+ a << (ptr / div.to_f).floor
235
+ ptr = ptr % div
236
+ end
237
+
238
+ # remap based on perm
239
+ remaped = perm.map { |x| index[x] }
240
+
241
+ ptr2 = 0
242
+ multipliers.each_with_index do |m, idx|
243
+ ptr2 += remaped[idx] * m
244
+ end
245
+
246
+ new_arr[ptr2] = arr[p]
247
+ end
248
+
249
+ [new_arr, new_shape]
250
+ end
251
+
252
+ def reduce_axis(current_axis, axis, val, keep_dims, f)
253
+ return val unless val.is_a?(Array)
254
+
255
+ r = val.collect do |v|
256
+ reduce_axis(current_axis + 1, axis, v, keep_dims, f)
257
+ end
258
+
259
+ should_reduce_axis = axis.nil? || (axis.is_a?(Array) && axis.include?(current_axis)) || (current_axis == axis)
260
+
261
+ if should_reduce_axis
262
+ reduced_val = r[0]
263
+ if r.size > 1
264
+ reduced_val = f.call(r[0..val.size])
265
+ elsif r.empty?
266
+ reduced_val = f.call(nil)
267
+ end
268
+ keep_dims ? [reduced_val] : reduced_val
269
+ else
270
+ r
271
+ end
272
+ end
273
+
274
+ def reduce(val, axis, keep_dims, func = nil)
275
+ rank = get_rank(val)
276
+ return val if axis && axis.is_a?(Array) && axis.empty?
277
+
278
+ func = lambda do |arr|
279
+ reduced_val = arr[0]
280
+ arr[1..arr.size].each do |v|
281
+ reduced_val = vector_op(reduced_val, v, ->(t, u) { t + u })
282
+ end
283
+ reduced_val
284
+ end if func.nil?
285
+
286
+ axis = if axis.nil?
287
+ nil
288
+ elsif axis.is_a?(Array)
289
+ return val if axis.empty?
290
+
291
+ axis.map { |a| a < 0 ? rank - a.abs : a }
292
+ else
293
+ axis < 0 ? rank - axis.abs : axis
294
+ end
295
+
296
+ reduce_axis(0, axis, val, keep_dims, func)
297
+ end
208
298
  end
209
299
  end
@@ -139,16 +139,16 @@ module TensorStream
139
139
  call_op(:sign, inputs[0], context, func)
140
140
  end
141
141
 
142
- register_op(:logical_and) do |context, _tensor, inputs|
143
- call_vector_op(:logical_and, inputs[0], inputs[1], context, ->(t, u) { t && u })
142
+ register_op(:logical_and) do |context, tensor, inputs|
143
+ call_vector_op(tensor, :logical_and, inputs[0], inputs[1], context, ->(t, u) { t && u })
144
144
  end
145
145
 
146
- register_op(:equal) do |context, _tensor, inputs|
147
- call_vector_op(:equal, inputs[0], inputs[1], context, ->(t, u) { t == u })
146
+ register_op(:equal) do |context, tensor, inputs|
147
+ call_vector_op(tensor, :equal, inputs[0], inputs[1], context, ->(t, u) { t == u })
148
148
  end
149
149
 
150
- register_op(:not_equal) do |context, _tensor, inputs|
151
- call_vector_op(:not_equal, inputs[0], inputs[1], context, ->(t, u) { t != u })
150
+ register_op(:not_equal) do |context, tensor, inputs|
151
+ call_vector_op(tensor, :not_equal, inputs[0], inputs[1], context, ->(t, u) { t != u })
152
152
  end
153
153
 
154
154
  register_op :index, no_eval: true do |_context, _tensor, inputs|
@@ -179,28 +179,80 @@ module TensorStream
179
179
  end
180
180
  end
181
181
 
182
- register_op %i[flow_dynamic_stitch dynamic_stitch], noop: true do |_context, _tensor, inputs|
182
+ register_op %i[flow_dynamic_stitch dynamic_stitch] do |_context, _tensor, inputs|
183
183
  indexes, data = inputs
184
184
  merged = []
185
185
  merge_dynamic_stitch(merged, indexes, data)
186
186
  merged
187
187
  end
188
188
 
189
+ register_op :gather do |_context, _tensor, inputs|
190
+ params, indexes = inputs
191
+ gather(params, indexes)
192
+ end
193
+
194
+ register_op :setdiff1d do |_context, tensor, inputs|
195
+ input, remove = inputs
196
+ idx = []
197
+ out = []
198
+ input.each_with_index do |x, index|
199
+ next if remove.include?(x)
200
+ out << x
201
+ idx << index
202
+ end
203
+ idx = idx.map { |i| Tensor.cast_dtype(i, tensor.options[:index_dtype]) } unless tensor.options[:index_dtype] == :int32
204
+ OutputGroup.new([out, idx], tensor.inputs.map(&:data_type))
205
+ end
206
+
207
+ register_op :cumprod do |context, tensor, inputs|
208
+ x = inputs[0]
209
+ c = fp_type?(tensor.data_type) ? 1.0 : 1
210
+ reverse_option = tensor.options[:reverse]
211
+ exclusive = tensor.options[:exclusive]
212
+
213
+ func = lambda do |arr|
214
+ return c if arr.nil?
215
+ count = arr.size
216
+
217
+
218
+ arr = arr.reverse if reverse_option
219
+ arr = [1] + arr if exclusive
220
+
221
+ start_prod = arr[0]
222
+ mapped = arr[1...count].map do |v|
223
+ start_prod = vector_op(start_prod, v, ->(a, b) { a * b })
224
+ end
225
+
226
+ arr = [arr[0]] + mapped
227
+ reverse_option ? arr.reverse : arr
228
+ end
229
+ reduction(context, tensor, func)
230
+ end
231
+
232
+ register_op :invert_permutation do |_context, _tensor, inputs|
233
+ input = inputs[0]
234
+ output = input.dup
235
+ input.size.times.each do |index|
236
+ output[input[index]] = index
237
+ end unless input.nil?
238
+ output
239
+ end
240
+
189
241
  register_op :size do |_context, tensor, inputs|
190
242
  input = inputs[0]
191
243
  Tensor.cast_dtype(input.flatten.size, tensor.options[:out_type])
192
244
  end
193
245
 
194
- register_op %i[neg negate], no_eval: true do |context, _tensor, inputs|
195
- call_vector_op(:negate, inputs[0], nil, context, ->(t, _u) { -t })
246
+ register_op %i[neg negate], no_eval: true do |context, tensor, inputs|
247
+ call_vector_op(tensor, :negate, inputs[0], nil, context, ->(t, _u) { -t })
196
248
  end
197
249
 
198
- register_op :add, no_eval: true do |context, _tensor, inputs|
250
+ register_op :add, no_eval: true do |context, tensor, inputs|
199
251
  a, b = inputs
200
- call_vector_op(:add, a, b, context, ->(t, u) { t + u })
252
+ call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
201
253
  end
202
254
 
203
- register_op :add_n, no_eval: true do |context, _tensor, inputs|
255
+ register_op :add_n, no_eval: true do |context, tensor, inputs|
204
256
  if inputs.size == 1
205
257
  complete_eval(inputs[0], context)
206
258
  elsif inputs.size > 1
@@ -208,48 +260,48 @@ module TensorStream
208
260
  a = inputs.pop
209
261
  until inputs.empty?
210
262
  b = inputs.pop
211
- a = call_vector_op(:add, a, b, context, ->(t, u) { t + u })
263
+ a = call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
212
264
  end
213
265
  a
214
266
  end
215
267
  end
216
268
 
217
- register_op :sub, no_eval: true do |context, _tensor, inputs|
269
+ register_op :sub, no_eval: true do |context, tensor, inputs|
218
270
  a, b = inputs
219
- call_vector_op(:sub, a, b, context, ->(t, u) { t - u })
271
+ call_vector_op(tensor, :sub, a, b, context, ->(t, u) { t - u })
220
272
  end
221
273
 
222
- register_op %i[floor_mod mod], no_eval: true do |context, _tensor, inputs|
274
+ register_op %i[floor_mod mod], no_eval: true do |context, tensor, inputs|
223
275
  a, b = inputs
224
- call_vector_op(:sub, a, b, context, ->(t, u) { t % u })
276
+ call_vector_op(tensor, :mod, a, b, context, ->(t, u) { t % u })
225
277
  end
226
278
 
227
- register_op %i[floor_div real_div], no_eval: true do |context, tensor, inputs|
279
+ register_op %i[floor_div], no_eval: true do |context, tensor, inputs|
228
280
  a, b = inputs
229
281
  if fp_type?(tensor.data_type)
230
- call_vector_op(:sub, a, b, context, ->(t, u) { (t / u).to_i.to_f })
282
+ call_vector_op(tensor, :div, a, b, context, ->(t, u) { (t / u).to_i.to_f })
231
283
  else
232
- call_vector_op(:sub, a, b, context, ->(t, u) { t / u })
284
+ call_vector_op(tensor, :div, a, b, context, ->(t, u) { t / u })
233
285
  end
234
286
  end
235
287
 
236
- register_op :mul, no_eval: true do |context, _tensor, inputs|
288
+ register_op :mul, no_eval: true do |context, tensor, inputs|
237
289
  a, b = inputs
238
- call_vector_op(:mul, a, b, context, ->(t, u) { t * u })
290
+ call_vector_op(tensor, :mul, a, b, context, ->(t, u) { t * u })
239
291
  end
240
292
 
241
- register_op :pow, no_eval: true do |context, _tensor, inputs|
293
+ register_op :pow, no_eval: true do |context, tensor, inputs|
242
294
  a, b = inputs
243
- call_vector_op(:pow, a, b, context, ->(t, u) { t**u })
295
+ call_vector_op(tensor, :pow, a, b, context, ->(t, u) { t**u })
244
296
  end
245
297
 
246
- register_op :squared_difference, no_eval: true do |context, _tensor, inputs|
298
+ register_op :squared_difference, no_eval: true do |context, tensor, inputs|
247
299
  a, b = inputs
248
- call_vector_op(:squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
300
+ call_vector_op(tensor, :squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
249
301
  end
250
302
 
251
303
  register_op %i[concat concat_v2] do |_context, tensor, inputs|
252
- concat_array(inputs[0], tensor.options[:axis])
304
+ concat_array(inputs, tensor.options[:axis])
253
305
  end
254
306
 
255
307
  register_op :round, no_eval: true do |context, _tensor, inputs|
@@ -268,6 +320,10 @@ module TensorStream
268
320
  call_op(:tan, inputs[0], context, ->(t, _b) { Math.tan(t) })
269
321
  end
270
322
 
323
+ register_op :atan, no_eval: true do |context, _tensor, inputs|
324
+ call_op(:atan, inputs[0], context, ->(t, _b) { Math.atan(t) })
325
+ end
326
+
271
327
  register_op :sec, no_eval: true do |context, _tensor, inputs|
272
328
  call_op(:sec, inputs[0], context, ->(t, _b) { Math.sec(t) })
273
329
  end
@@ -328,9 +384,9 @@ module TensorStream
328
384
  inputs[0]
329
385
  end
330
386
 
331
- register_op :sigmoid_grad, no_eval: true do |context, _tensor, inputs|
387
+ register_op :sigmoid_grad, no_eval: true do |context, tensor, inputs|
332
388
  a, b = inputs
333
- call_vector_op(:sigmoid_grad, a, b, context, ->(t, u) { u * sigmoid(t) * (1 - sigmoid(t)) })
389
+ call_vector_op(tensor, :sigmoid_grad, a, b, context, ->(t, u) { u * sigmoid(t) * (1 - sigmoid(t)) })
334
390
  end
335
391
 
336
392
  register_op :random_uniform, no_eval: true do |_context, tensor, _inputs|
@@ -378,20 +434,72 @@ module TensorStream
378
434
 
379
435
  register_op :assign, noop: true do |context, tensor, _inputs|
380
436
  assign = tensor.inputs[0] || tensor
381
- assign.value = complete_eval(tensor.inputs[1], context)
437
+ assign.value = global_eval(tensor, tensor.inputs[1], context)
382
438
  assign.value
383
439
  end
384
440
 
385
441
  register_op :assign_add, noop: true do |context, tensor, _inputs|
386
- tensor.inputs[0].value = process_vector_math_op(tensor.inputs[0], tensor.inputs[1], context, ->(t, u) { t + u })
442
+ tensor.inputs[0].value = process_vector_math_op(tensor, tensor.inputs[0], tensor.inputs[1], context, ->(t, u) { t + u })
443
+ tensor.inputs[0].value
444
+ end
445
+
446
+ register_op :variable, noop: true do |_context, tensor, _inputs|
387
447
  tensor.inputs[0].value
388
448
  end
389
449
 
390
450
  register_op :assign_sub, noop: true do |context, tensor, _inputs|
391
- tensor.inputs[0].value = process_vector_math_op(tensor.inputs[0], tensor.inputs[1], context, ->(t, u) { t - u })
451
+ tensor.inputs[0].value = process_vector_math_op(tensor, tensor.inputs[0], tensor.inputs[1], context, ->(t, u) { t - u })
392
452
  tensor.inputs[0].value
393
453
  end
394
454
 
455
+ register_op :stack do |_context, tensor, inputs|
456
+ axis = tensor.options[:axis] || 0
457
+ shape = shape_eval(inputs[0])
458
+ rank = shape.size + 1
459
+ elem_size = shape.empty? ? 1 : shape.reduce(:*)
460
+ output_buffer = Array.new(inputs.size * elem_size) { 0 }
461
+ new_shape = [inputs.size]
462
+ shape.inject(new_shape) { |ns, s| ns << s }
463
+
464
+ divisors = new_shape.dup.drop(1).reverse.inject([1]) do |a, s|
465
+ a << s * a.last
466
+ end.reverse
467
+
468
+ axis = rank + axis if axis < 0
469
+ rotated_shape = Array.new(axis + 1) { new_shape.shift }
470
+ new_shape = rotated_shape.rotate! + new_shape
471
+
472
+ multipliers = new_shape.dup.drop(1).reverse.inject([1]) do |a, s|
473
+ a << s * a.last
474
+ end.reverse
475
+
476
+ inputs.each_with_index do |input, index|
477
+ raw_input = input.is_a?(Array) ? input.flatten : [input]
478
+ start = index * divisors.first
479
+
480
+ raw_input.each_with_index do |x, index2|
481
+ index_map = []
482
+ ptr = start + index2
483
+ divisors.each_with_object(index_map) do |div, a|
484
+ a << (ptr / div.to_f).floor
485
+ ptr = ptr % div
486
+ end
487
+
488
+ rotated_index = Array.new(axis + 1) { index_map.shift }
489
+ index_map = rotated_index.rotate! + index_map
490
+
491
+ ptr2 = 0
492
+ multipliers.each_with_index do |m, idx|
493
+ ptr2 += index_map[idx] * m
494
+ end
495
+
496
+ output_buffer[ptr2] = x
497
+ end
498
+ end
499
+
500
+ TensorShape.reshape(output_buffer, new_shape)
501
+ end
502
+
395
503
  register_op :mean, noop: true do |context, tensor, _inputs|
396
504
  c = fp_type?(tensor.data_type) ? 0.0 : 0
397
505
  func = lambda do |arr|
@@ -473,8 +581,20 @@ module TensorStream
473
581
  call_op(:tanh_grad, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
474
582
  end
475
583
 
476
- register_op :transpose do |_context, _tensor, inputs|
477
- inputs[0].transpose
584
+ register_op :transpose do |_context, tensor, inputs|
585
+ shape = shape_eval(inputs[0])
586
+ rank = get_rank(inputs[0])
587
+ perm = inputs[1] || (0...rank).to_a.reverse
588
+ if rank == 2 && perm.nil? # use native transpose for general case
589
+ inputs[0].transpose
590
+ else
591
+ arr = inputs[0].flatten
592
+
593
+ new_shape = perm.map { |p| shape[p] }
594
+ new_arr = Array.new(shape.reduce(:*)) { 0 }
595
+ transpose_with_perm(arr, new_arr, shape, new_shape, perm)
596
+ TensorShape.reshape(new_arr, new_shape)
597
+ end
478
598
  end
479
599
 
480
600
  register_op :eye do |_context, tensor, inputs|
@@ -491,7 +611,7 @@ module TensorStream
491
611
  end
492
612
  end
493
613
 
494
- register_op :expand_dims do |context, tensor, inputs|
614
+ register_op :expand_dims do |_context, _tensor, inputs|
495
615
  val, axis = inputs
496
616
  axis = axis.nil? ? 0 : axis
497
617
 
@@ -504,12 +624,12 @@ module TensorStream
504
624
  end
505
625
 
506
626
  register_op :cond, noop: true do |context, tensor, inputs|
507
- pred = complete_eval(tensor.options[:pred], context)
627
+ pred = global_eval(tensor, tensor.options[:pred], context)
508
628
 
509
629
  if all_true?(pred)
510
- complete_eval(inputs[0], context)
630
+ global_eval(tensor, inputs[0], context)
511
631
  else
512
- complete_eval(inputs[1], context)
632
+ global_eval(tensor, inputs[1], context)
513
633
  end
514
634
  end
515
635
 
@@ -518,24 +638,24 @@ module TensorStream
518
638
  call_3way_vector_op(pred, inputs[0], inputs[1], context, ->(t, u, v) { t ? u : v })
519
639
  end
520
640
 
521
- register_op :less do |context, _tensor, inputs|
641
+ register_op :less do |context, tensor, inputs|
522
642
  a, b = inputs
523
- call_vector_op(:less, a, b, context, ->(t, u) { t < u })
643
+ call_vector_op(tensor, :less, a, b, context, ->(t, u) { t < u })
524
644
  end
525
645
 
526
- register_op :greater do |context, _tensor, inputs|
646
+ register_op :greater do |context, tensor, inputs|
527
647
  a, b = inputs
528
- call_vector_op(:greater, a, b, context, ->(t, u) { t > u })
648
+ call_vector_op(tensor, :greater, a, b, context, ->(t, u) { t > u })
529
649
  end
530
650
 
531
- register_op :greater_equal do |context, _tensor, inputs|
651
+ register_op :greater_equal do |context, tensor, inputs|
532
652
  a, b = inputs
533
- call_vector_op(:greater_equal, a, b, context, ->(t, u) { t >= u })
653
+ call_vector_op(tensor, :greater_equal, a, b, context, ->(t, u) { t >= u })
534
654
  end
535
655
 
536
- register_op :less_equal do |context, _tensor, inputs|
656
+ register_op :less_equal do |context, tensor, inputs|
537
657
  a, b = inputs
538
- call_vector_op(:greater_equal, a, b, context, ->(t, u) { t <= u })
658
+ call_vector_op(tensor, :greater_equal, a, b, context, ->(t, u) { t <= u })
539
659
  end
540
660
 
541
661
  register_op :fill do |_context, _tensor, inputs|
@@ -622,8 +742,8 @@ module TensorStream
622
742
  get_rank(inputs[0])
623
743
  end
624
744
 
625
- register_op :div, noop: true do |context, _tensor, inputs|
626
- process_vector_math_op(inputs[0], inputs[1], context, ->(t, u) { t / u })
745
+ register_op %i[div real_div], noop: true do |context, tensor, inputs|
746
+ process_vector_math_op(tensor, inputs[0], inputs[1], context, ->(t, u) { t / u })
627
747
  end
628
748
 
629
749
  register_op :reshape do |_context, _tensor, inputs|
@@ -646,31 +766,44 @@ module TensorStream
646
766
  arr_pad(inputs[0], p, tensor.data_type)
647
767
  end
648
768
 
649
- register_op %i[max maximum], noop: true do |context, _tensor, inputs|
650
- call_vector_op(:max, inputs[0], inputs[1], context, ->(t, u) { [t, u].max })
769
+ register_op %i[max maximum], noop: true do |context, tensor, inputs|
770
+ call_vector_op(tensor, :max, inputs[0], inputs[1], context, ->(t, u) { [t, u].max })
651
771
  end
652
772
 
653
- register_op %i[min minimum], noop: true do |context, _tensor, inputs|
654
- call_vector_op(:min, inputs[0], inputs[1], context, ->(t, u) { [t, u].min })
773
+ register_op %i[min minimum], noop: true do |context, tensor, inputs|
774
+ call_vector_op(tensor, :min, inputs[0], inputs[1], context, ->(t, u) { [t, u].min })
655
775
  end
656
776
 
657
777
  register_op :apply_gradient_descent do |context, tensor, inputs|
658
778
  target_var, learning_rate, delta = inputs
659
779
  assign = tensor.inputs[0] || tensor
660
780
 
661
- assign.value = process_vector_math_op(target_var, delta, context, ->(t, u) { t - u * learning_rate })
781
+ assign.value = process_vector_math_op(tensor, target_var, delta, context, ->(t, u) { t - u * learning_rate })
782
+ assign.value
783
+ end
784
+
785
+ register_op :apply_momentum do |context, tensor, inputs|
786
+ target_var, momentum_var, learning_rate, grad, momentum = inputs
787
+ assign = tensor.inputs[0] || tensor
788
+ assign_acc = tensor.inputs[1]
789
+ assign_acc.value = process_vector_math_op(tensor, momentum_var, grad, context, ->(t, u) { t * momentum + u })
790
+ if tensor.options[:use_nesterov]
791
+ delta = process_vector_math_op(tensor, grad, momentum_var, context, ->(g, acc) { g * learning_rate + acc * momentum * learning_rate })
792
+ assign.value = process_vector_math_op(tensor,target_var, delta, context, ->(t, u) { t - u })
793
+ else
794
+ assign.value = process_vector_math_op(tensor, target_var, momentum_var, context, ->(v, acc) { v - acc * learning_rate })
795
+ end
662
796
  assign.value
663
797
  end
664
798
 
665
- register_op :broadcast_gradient_args do |_context, _tensor, inputs|
799
+ register_op :broadcast_gradient_args do |_context, tensor, inputs|
666
800
  rx, ry = get_broadcast_gradient_args(inputs[0], inputs[1])
667
- OutputGroup.new([rx, ry])
801
+ OutputGroup.new([rx, ry], tensor.inputs.map(&:data_type))
668
802
  end
669
803
 
670
804
  register_op :tile do |_context, _tensor, inputs|
671
805
  input, multiples = inputs
672
806
  rank = get_rank(input)
673
-
674
807
  raise '1D or higher tensor required' if rank.zero?
675
808
  raise "invalid multiple size passed #{rank} != #{multiples.size}" if rank != multiples.size
676
809
 
@@ -678,8 +811,8 @@ module TensorStream
678
811
  tile.nil? ? [] : tile
679
812
  end
680
813
 
681
- register_op :flow_group, noop: true do |context, _tensor, inputs|
682
- inputs.collect { |input| run(input, context) }
814
+ register_op :flow_group, noop: true do |context, tensor, inputs|
815
+ inputs.collect { |input| global_eval(tensor, input, context) }
683
816
  end
684
817
 
685
818
  register_op :softmax do |_context, _tensor, inputs|
@@ -687,11 +820,11 @@ module TensorStream
687
820
  end
688
821
 
689
822
  register_op :save_v2 do |context, tensor, inputs|
690
- prefix, tensor_names, shape_and_slices = inputs[0..3]
823
+ # prefix, tensor_names, shape_and_slices = inputs[0..3]
691
824
  end
692
825
 
693
826
  register_op :restore_v2 do |context, tensor, inputs|
694
- prefix, tensor_names, shape_and_slices = inputs[0..3]
827
+ # prefix, tensor_names, shape_and_slices = inputs[0..3]
695
828
  end
696
829
 
697
830
  register_op :softmax_grad do |_context, _tensor, inputs|
@@ -744,24 +877,36 @@ module TensorStream
744
877
  end
745
878
  end
746
879
 
747
- register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context, _tensor, inputs|
880
+ register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context, tensor, inputs|
748
881
  last_dimen_list = last_axis(inputs[0])
749
882
  input_shape = shape_eval(inputs[0])
883
+ rank = input_shape.size - 1
750
884
  labels = last_axis(inputs[1])
751
885
  func = lambda { |logits, label|
752
886
  c = logits.max
753
887
  transformed_logits = logits.map { |l| l - c }
754
888
  sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
755
- transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
889
+ losses = transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
890
+ probs = transformed_logits.zip(label).map { |x, y| (Math.exp(x) / sum) - y }
891
+ [losses, probs]
756
892
  }
757
893
 
758
894
  if input_shape.size == 1
759
- func.call(last_dimen_list, labels)
895
+ loss, prob = func.call(last_dimen_list, labels)
896
+ loss = reduce(loss, rank, false)
897
+ OutputGroup.new([loss, prob], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
760
898
  else
761
- arr = last_dimen_list.zip(labels).collect do |list, label|
762
- func.call(list, label)
899
+ losses = []
900
+ backprobs = []
901
+ arr = last_dimen_list.zip(labels).each do |list, label|
902
+ loss, prob = func.call(list, label)
903
+ losses << loss
904
+ backprobs << prob
763
905
  end
764
- TensorShape.reshape(arr.flatten, input_shape)
906
+ reshaped_losses = TensorShape.reshape(losses.flatten, input_shape)
907
+ reshaped_backprops = TensorShape.reshape(backprobs.flatten, input_shape)
908
+ reshaped_losses = reduce(reshaped_losses, rank, false)
909
+ OutputGroup.new([reshaped_losses, reshaped_backprops], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
765
910
  end
766
911
  end
767
912
 
@@ -791,7 +936,7 @@ module TensorStream
791
936
  register_op :check_numerics do |context, tensor, inputs|
792
937
  message = tensor.options[:message]
793
938
  f = lambda { |t, _b|
794
- raise "#{message} Invalid argument" if t.nan? || t.infinite?
939
+ raise TensorStream::InvalidArgumentError, "#{message} Invalid argument" if t.nan? || t.infinite?
795
940
  t
796
941
  }
797
942
  call_op(:check_numerics, inputs[0], context, f)
@@ -799,13 +944,9 @@ module TensorStream
799
944
 
800
945
  def eval_operation(tensor, child_context)
801
946
  return @context[tensor.name] if @context.key?(tensor.name)
947
+ # puts "ruby: #{tensor.name}"
802
948
  invoke(tensor, child_context).tap do |result|
803
- # puts tensor.name
804
- # if tensor.name.start_with?('softmax_cross_entropy_with_logits')
805
- # puts result.inspect
806
- # end
807
- # result.flatten.each do |a|
808
- # end if result.is_a?(Array)
949
+
809
950
  if tensor.breakpoint
810
951
  a = resolve_placeholder(tensor.inputs[0], child_context) if tensor.inputs && tensor.inputs[0]
811
952
  b = resolve_placeholder(tensor.inputs[1], child_context) if tensor.inputs && tensor.inputs[1]
@@ -826,20 +967,21 @@ module TensorStream
826
967
  @context[tensor.name] = result
827
968
  end
828
969
  rescue EvaluatorExcecutionException => e
829
- raise e
970
+ raise e, "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true, 1)} defined at #{tensor.source}"
830
971
  rescue TensorStreamError => e
831
- raise e
972
+ raise e, "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true, 1)} defined at #{tensor.source}"
832
973
  rescue StandardError => e
833
- # a = resolve_placeholder(tensor.inputs[0], child_context) if tensor.inputs && tensor.inputs[0]
834
- # b = resolve_placeholder(tensor.inputs[1], child_context) if tensor.inputs && tensor.inputs[1]
974
+
975
+ a = resolve_placeholder(tensor.inputs[0], child_context) if tensor.inputs && tensor.inputs[0]
976
+ b = resolve_placeholder(tensor.inputs[1], child_context) if tensor.inputs && tensor.inputs[1]
835
977
  puts e.message
836
978
  puts e.backtrace.join("\n")
837
979
  # shape_a = a.shape.shape if a
838
980
  # shape_b = b.shape.shape if b
839
981
  # dtype_a = a.data_type if a
840
982
  # dtype_b = b.data_type if b
841
- # a = complete_eval(a, child_context)
842
- # b = complete_eval(b, child_context)
983
+ a = complete_eval(a, child_context)
984
+ b = complete_eval(b, child_context)
843
985
  # puts "name: #{tensor.given_name}"
844
986
  # # puts "op: #{tensor.to_math(true, 1)}"
845
987
  # puts "A #{shape_a} #{dtype_a}: #{a}" if a
@@ -909,23 +1051,10 @@ module TensorStream
909
1051
  end
910
1052
 
911
1053
  def reduction(child_context, tensor, func)
912
- val = complete_eval(tensor.inputs[0], child_context)
913
- axis = complete_eval(tensor.inputs[1], child_context)
914
- keep_dims = complete_eval(tensor.options[:keepdims], child_context)
915
- rank = get_rank(val)
916
- return val if axis && axis.is_a?(Array) && axis.empty?
917
-
918
- axis = if axis.nil?
919
- nil
920
- elsif axis.is_a?(Array)
921
- return val if axis.empty?
922
-
923
- axis.map { |a| a < 0 ? rank - a.abs : a }
924
- else
925
- axis < 0 ? rank - axis.abs : axis
926
- end
927
-
928
- reduce_axis(0, axis, val, keep_dims, func)
1054
+ val = global_eval(tensor, tensor.inputs[0], child_context)
1055
+ axis = global_eval(tensor, tensor.inputs[1], child_context)
1056
+ keep_dims = global_eval(tensor, tensor.options[:keepdims], child_context)
1057
+ reduce(val, axis, keep_dims, func)
929
1058
  end
930
1059
 
931
1060
  def arr_pad(arr, paddings, data_type = :float32, rank = 0)
@@ -960,15 +1089,15 @@ module TensorStream
960
1089
  TensorStream.send(op.to_sym, a)
961
1090
  end
962
1091
 
963
- def call_vector_op(op, a, b, child_context, func)
964
- process_vector_math_op(a, b, child_context, func)
1092
+ def call_vector_op(tensor, op, a, b, child_context, func)
1093
+ process_vector_math_op(tensor, a, b, child_context, func)
965
1094
  rescue FullEvalNotPossible
966
1095
  TensorStream.send(op.to_sym, a, b)
967
1096
  end
968
1097
 
969
- def process_vector_math_op(a, b, child_context, op)
970
- eval_a = complete_eval(a, child_context) unless a.nil?
971
- eval_b = complete_eval(b, child_context) unless b.nil?
1098
+ def process_vector_math_op(tensor, a, b, child_context, op)
1099
+ eval_a = global_eval(tensor, a, child_context) unless a.nil?
1100
+ eval_b = global_eval(tensor, b, child_context) unless b.nil?
972
1101
 
973
1102
  raise FullEvalNotPossible.new, "full eval not possible for #{a.name}" if eval_a.is_a?(Tensor) || eval_b.is_a?(Tensor)
974
1103
 
@@ -1025,28 +1154,6 @@ module TensorStream
1025
1154
  Tensor.cast_dtype(var, placeholder.data_type)
1026
1155
  end
1027
1156
 
1028
- def reduce_axis(current_axis, axis, val, keep_dims, f = ->(a, b) { a + b })
1029
- return val unless val.is_a?(Array)
1030
-
1031
- r = val.collect do |v|
1032
- reduce_axis(current_axis + 1, axis, v, keep_dims, f)
1033
- end
1034
-
1035
- should_reduce_axis = axis.nil? || (axis.is_a?(Array) && axis.include?(current_axis)) || (current_axis == axis)
1036
-
1037
- if should_reduce_axis
1038
- reduced_val = r[0]
1039
- if r.size > 1
1040
- reduced_val = f.call(r[0..val.size])
1041
- elsif r.empty?
1042
- reduced_val = f.call(nil)
1043
- end
1044
- keep_dims ? [reduced_val] : reduced_val
1045
- else
1046
- r
1047
- end
1048
- end
1049
-
1050
1157
  # handle 3 tensor math operations
1051
1158
  def call_3way_vector_op(v_a, v_b, v_c, child_context, op = ->(a, b, c) { a + b + c })
1052
1159
  return op.call(v_a, v_b, v_c) unless v_a.is_a?(Array)