tensor_stream 0.8.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: f84c2b9852fcf4931c47c0130b67497a50a87b0f
4
- data.tar.gz: 524e1105da4e06e3472cbcfa0e6f764ae4512d37
2
+ SHA256:
3
+ metadata.gz: a3c7d0a810a79ceedc0237379b105d7a9b598ba2513ef2d59ba3cec78d7b0da0
4
+ data.tar.gz: 7c0d90b27e548b72a86e88e7181f3d3b131a5fa3c6800000c743dd6e47d47b3b
5
5
  SHA512:
6
- metadata.gz: 420e2675ab67d4c8462534bdf8c703671656f7852d984579e22ee57f1425dd5740fcb64a1e52363bf337cd7c691d87a75c76bd868b13c8a7f06d78e0eb00aa73
7
- data.tar.gz: 24fe1022741883d46cdd5af51309da33d421d72874f0cc84bf2e0ed14a62602f1830c6060bd86e42359b7962b4a57727c9a48ce13d5950d5ba02f6a9cdfd719f
6
+ metadata.gz: a7a8a5607883d868da3ceaa9f870ac5c6d6809d45992a9ae0e4f26d5648bda2c4a26ea3abec506abd92362dcf3b63935b3f6acbcc70b605600307313f8c69f49
7
+ data.tar.gz: db8917bee53f91e1017b5fdb8b9ece8b50a1096d249cee40eed8c335f597f32e5ce81056230b8ce1b9acff4df728497fa13e896212cbdaccc023dbdb7ed3591e
data/CHANGELOG.md CHANGED
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
4
4
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
5
5
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
6
 
7
+
8
+ ## [0.8.1] - 2018-08-30
9
+ - [TRAINING] Added AdamOptimizer
10
+
7
11
  ## [0.8.0] - 2018-08-29
8
12
  ### Added
9
13
  - [TRAINING] Added new supported optimizer, MomentumOptimizer loosely based on tensorflow's implementation (with nesterov support)
data/README.md CHANGED
@@ -17,7 +17,20 @@ The goal of this gem is to have a high performance machine learning and compute
17
17
  - eager execution (experimental)
18
18
  - (08-08-2018) Load pbtext files from tensorflow (Graph.parse_from_string)
19
19
 
20
- Since this is a pure ruby implementation for now, performance is not there yet. However it should be a good enough environment to learn about tensorflow and experiment with some models.
20
+ ## Compatibility
21
+
22
+ TensorStream comes with a pure ruby and OpenCL implementation out of the box. The pure ruby implementation
23
+ is known to work with most ruby implementations including TruffleRuby, JRuby as well as jit enabled versions of mri (ruby-2.6.0).
24
+
25
+ OpenCL is supported only on mri implementations of ruby. This can be enabled by including the OpenCL evaluator (Make sure you have OpenCL drivers installed correctly on your system):
26
+
27
+ ```ruby
28
+ require 'tensor_stream/evaluator/opencl/opencl_evaluator'
29
+ ```
30
+
31
+ OpenCL is basically a requirement for deep learning and image processing tasks as the ruby implementation is too slow even with jit speedups using latest ruby implementations.
32
+
33
+ OpenCL kernels used by tensorstream can be found at tensor_stream/lib/evaluator/opencl/kernels. These are non specific and should work with any device that supports OpenCL including intel GPUs and CPUs, as well as GPUs from Nvidia and AMD.
21
34
 
22
35
  ## Installation
23
36
 
@@ -72,6 +85,8 @@ pred = X * W + b
72
85
  # Mean squared error
73
86
  cost = ((pred - Y) ** 2).reduce(:+) / ( 2 * n_samples)
74
87
 
88
+ # optimizer = TensorStream::Train::MomentumOptimizer.new(0.01, 0.5, use_nesterov: true).minimize(cost)
89
+ # optimizer = TensorStream::Train::AdamOptimizer.new.minimize(cost)
75
90
  optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate).minimize(cost)
76
91
 
77
92
  # Initialize the variables (i.e. assign their default value)
@@ -338,7 +353,7 @@ ruby 2.4
338
353
  $ ruby -v
339
354
  ruby 2.4.0p0 (2016-12-24 revision 57164) [x86_64-linux]
340
355
  $ ruby samples/linear_regression.rb
341
- 495 seconds 1000 epochs
356
+ 495 seconds 10000 epochs
342
357
  ```
343
358
 
344
359
  ruby 2.6.0-preview2
@@ -383,8 +398,6 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
383
398
 
384
399
  Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/tensor_stream. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
385
400
 
386
-
387
401
  ## License
388
402
 
389
- The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
390
-
403
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
@@ -0,0 +1,23 @@
1
+ % c_dtype = dtype_to_c_type(dtype)
2
+ // same dimension add floating point op
3
+ __kernel void apply_adam_<%= dtype %>(const int M, const int N,
4
+ __global const <%= c_dtype %> *grad,
5
+ __global const <%= c_dtype %> *learning_rate,
6
+ __global const <%= c_dtype %> *beta1_power,
7
+ __global const <%= c_dtype %> *beta2_power,
8
+ __global const <%= c_dtype %> *beta1,
9
+ __global const <%= c_dtype %> *beta2,
10
+ __global const <%= c_dtype %> *epsilon,
11
+ __global <%= c_dtype %> *momentum,
12
+ __global <%= c_dtype %> *output, __global <%= c_dtype %> *v) {
13
+ // Get the index of the current element to be processed
14
+ const int globalRow = get_global_id(0); // Row ID of C (0..M)
15
+ const int globalCol = get_global_id(1); // Col ID of C (0..N)
16
+ const int index = globalRow * N + globalCol;
17
+
18
+ <%= c_dtype %> alpha = learning_rate[0] * sqrt(1.0 - beta2_power[0]) / (1.0 - beta1_power[0]);
19
+
20
+ momentum[index] += (grad[index] - momentum[index]) * (1.0 - beta1[0]);
21
+ v[index] += (grad[index] * grad[index] - v[index]) * (1.0 - beta2[0]);
22
+ output[index] -= (momentum[index] * alpha) / ( sqrt(v[index]) + epsilon[0] );
23
+ }
@@ -107,9 +107,8 @@ module TensorStream
107
107
  end
108
108
  end
109
109
 
110
- def complete_eval(tensor, context)
110
+ def enqueue_buffer_read(tensor, context)
111
111
  buffer = _run(tensor, context)
112
-
113
112
  if buffer.is_a?(Array)
114
113
  buffer = buffer.collect do |b|
115
114
  next b if b.buffer.size.zero?
@@ -122,7 +121,12 @@ module TensorStream
122
121
  return [] if buffer.buffer.nil?
123
122
  return buffer if buffer.buffer.size.zero?
124
123
  _opencl_queue.enqueue_read_buffer(buffer.cl_buffer, buffer.buffer, event_wait_list: build_event_wait_list([buffer]))
124
+ buffer
125
125
  end
126
+ end
127
+
128
+ def complete_eval(tensor, context)
129
+ buffer = enqueue_buffer_read(tensor, context)
126
130
  _opencl_queue.finish
127
131
  buffer
128
132
  end
@@ -339,6 +343,48 @@ module TensorStream
339
343
  learning_rate.cl_buffer, momentum.cl_buffer, output_buffer.cl_buffer,
340
344
  assign_acc.buffer.cl_buffer, event_wait_list: event_wait_list)
341
345
  output_buffer.op = event
346
+ assign_acc.buffer.op = event
347
+ output_buffer
348
+ end
349
+
350
+ # Adam optimization algorithm
351
+ register_op :apply_adam do |_context, tensor, inputs|
352
+ _target_var, _m, _v, beta1_power, beta2_power, lr_t, beta1_t, beta2_t, epsilon_t, grad = inputs
353
+
354
+ assign = tensor.inputs[0] || tensor
355
+ assign_m = tensor.inputs[1]
356
+ assign_v = tensor.inputs[2]
357
+
358
+ # mark variable buffers as dirty
359
+ assign.buffer.dirty = true # force buffer copy when variable is read externally
360
+ assign_m.buffer.dirty = true # force buffer copy when variable is read externally
361
+ assign_v.buffer.dirty = true # force buffer copy when variable is read externally
362
+
363
+ output_buffer = assign.buffer
364
+
365
+ m, n = output_buffer.shape
366
+ work_group = [m || 1, n || 1]
367
+ cl_m = OpenCL::Int1.new(m || 1)
368
+ cl_n = OpenCL::Int1.new(n || 1)
369
+
370
+ event_wait_list = build_event_wait_list(inputs)
371
+ method_call = :"apply_adam_#{output_buffer.data_type}"
372
+ event = _cl_program("apply_adam", dtype: output_buffer.data_type)
373
+ .send(method_call, _opencl_queue, work_group, cl_m, cl_n,
374
+ grad.cl_buffer,
375
+ lr_t.cl_buffer,
376
+ beta1_power.cl_buffer,
377
+ beta2_power.cl_buffer,
378
+ beta1_t.cl_buffer,
379
+ beta2_t.cl_buffer,
380
+ epsilon_t.cl_buffer,
381
+ assign_m.buffer.cl_buffer,
382
+ assign.buffer.cl_buffer,
383
+ assign_v.buffer.cl_buffer,
384
+ event_wait_list: event_wait_list)
385
+ output_buffer.op = event
386
+ assign_m.buffer.op = event
387
+ assign_v.buffer.op = event
342
388
  output_buffer
343
389
  end
344
390
 
@@ -713,8 +759,9 @@ module TensorStream
713
759
  convert_to_opencl(arr.buffer, shape, data_type: arr.data_type, name: tensor.name)
714
760
  end
715
761
 
716
- register_op :flow_group do |_context, _tensor, inputs|
717
- inputs
762
+ register_op :flow_group do |context, _tensor, inputs|
763
+ _opencl_queue.finish
764
+ nil
718
765
  end
719
766
 
720
767
  register_op :size do |_context, tensor, inputs|
@@ -0,0 +1,144 @@
1
+ module TensorStream
2
+ module MathOps
3
+ def MathOps.included(klass)
4
+ klass.class_eval do
5
+ register_op :tanh, no_eval: true do |context, _tensor, inputs|
6
+ call_op(:tanh, inputs[0], context, ->(t, _b) { Math.tanh(t) })
7
+ end
8
+
9
+ register_op :tan, no_eval: true do |context, _tensor, inputs|
10
+ call_op(:tan, inputs[0], context, ->(t, _b) { Math.tan(t) })
11
+ end
12
+
13
+ register_op :atan, no_eval: true do |context, _tensor, inputs|
14
+ call_op(:atan, inputs[0], context, ->(t, _b) { Math.atan(t) })
15
+ end
16
+
17
+ register_op :sec, no_eval: true do |context, _tensor, inputs|
18
+ call_op(:sec, inputs[0], context, ->(t, _b) { Math.sec(t) })
19
+ end
20
+
21
+ register_op :sin, no_eval: true do |context, _tensor, inputs|
22
+ call_op(:sin, inputs[0], context, ->(t, _b) { Math.sin(t) })
23
+ end
24
+
25
+ register_op :add, no_eval: true do |context, tensor, inputs|
26
+ a, b = inputs
27
+ call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
28
+ end
29
+
30
+ register_op :add_n, no_eval: true do |context, tensor, inputs|
31
+ if inputs.size == 1
32
+ complete_eval(inputs[0], context)
33
+ elsif inputs.size > 1
34
+
35
+ a = inputs.pop
36
+ until inputs.empty?
37
+ b = inputs.pop
38
+ a = call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
39
+ end
40
+ a
41
+ end
42
+ end
43
+
44
+ register_op :sub, no_eval: true do |context, tensor, inputs|
45
+ a, b = inputs
46
+ call_vector_op(tensor, :sub, a, b, context, ->(t, u) { t - u })
47
+ end
48
+
49
+ register_op %i[floor_mod mod], no_eval: true do |context, tensor, inputs|
50
+ a, b = inputs
51
+ call_vector_op(tensor, :mod, a, b, context, ->(t, u) { t % u })
52
+ end
53
+
54
+ register_op %i[floor_div], no_eval: true do |context, tensor, inputs|
55
+ a, b = inputs
56
+ if fp_type?(tensor.data_type)
57
+ call_vector_op(tensor, :div, a, b, context, ->(t, u) { (t / u).to_i.to_f })
58
+ else
59
+ call_vector_op(tensor, :div, a, b, context, ->(t, u) { t / u })
60
+ end
61
+ end
62
+
63
+ register_op :mul, no_eval: true do |context, tensor, inputs|
64
+ a, b = inputs
65
+ call_vector_op(tensor, :mul, a, b, context, ->(t, u) { t * u })
66
+ end
67
+
68
+ register_op :pow, no_eval: true do |context, tensor, inputs|
69
+ a, b = inputs
70
+ call_vector_op(tensor, :pow, a, b, context, ->(t, u) { t**u })
71
+ end
72
+
73
+ register_op :squared_difference, no_eval: true do |context, tensor, inputs|
74
+ a, b = inputs
75
+ call_vector_op(tensor, :squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
76
+ end
77
+
78
+ register_op :round, no_eval: true do |context, _tensor, inputs|
79
+ call_op(:round, inputs[0], context, ->(t, _b) { t.round })
80
+ end
81
+
82
+ register_op :abs, no_eval: true do |context, _tensor, inputs|
83
+ call_op(:abs, inputs[0], context, ->(t, _b) { t.abs })
84
+ end
85
+
86
+ register_op :asin, no_eval: true do |context, _tensor, inputs|
87
+ call_op(:asin, inputs[0], context, ->(t, _b) { Math.asin(t) })
88
+ end
89
+
90
+ register_op :acos, no_eval: true do |context, _tensor, inputs|
91
+ call_op(:acos, inputs[0], context, ->(t, _b) { Math.acos(t) })
92
+ end
93
+
94
+ register_op :cos, no_eval: true do |context, _tensor, inputs|
95
+ call_op(:cos, inputs[0], context, ->(t, _b) { Math.cos(t) })
96
+ end
97
+
98
+ register_op :log1p, no_eval: true do |context, _tensor, inputs|
99
+ call_op(:log1p, inputs[0], context, ->(t, _b) { Math.log(1 + t) })
100
+ end
101
+
102
+ register_op :log, no_eval: true do |context, _tensor, inputs|
103
+ call_op(:log, inputs[0], context, ->(t, _b) { t < 0 ? Float::NAN : Math.log(t) })
104
+ end
105
+
106
+ register_op :exp, no_eval: true do |context, _tensor, inputs|
107
+ call_op(:exp, inputs[0], context, ->(t, _b) { Math.exp(t) })
108
+ end
109
+
110
+ register_op :sigmoid, no_eval: true do |context, _tensor, inputs|
111
+ call_op(:sigmoid, inputs[0], context, ->(t, _b) { sigmoid(t) })
112
+ end
113
+
114
+ register_op :sqrt, no_eval: true do |context, _tensor, inputs|
115
+ call_op(:sqrt, inputs[0], context, ->(t, _b) { Math.sqrt(t) })
116
+ end
117
+
118
+ register_op :floor, no_eval: true do |context, _tensor, inputs|
119
+ call_op(:floor, inputs[0], context, ->(t, _b) { t.floor })
120
+ end
121
+
122
+ register_op :ceil, no_eval: true do |context, _tensor, inputs|
123
+ call_op(:ceil, inputs[0], context, ->(t, _b) { t.ceil })
124
+ end
125
+
126
+ register_op :square, no_eval: true do |context, _tensor, inputs|
127
+ call_op(:square, inputs[0], context, ->(t, _b) { t * t })
128
+ end
129
+
130
+ register_op :reciprocal, no_eval: true do |context, _tensor, inputs|
131
+ call_op(:reciprocal, inputs[0], context, ->(t, _b) { 1 / t })
132
+ end
133
+
134
+ register_op %i[neg negate], no_eval: true do |context, tensor, inputs|
135
+ call_vector_op(tensor, :negate, inputs[0], nil, context, ->(t, _u) { -t })
136
+ end
137
+
138
+ register_op :tanh_grad, no_eval: true do |context, _tensor, inputs|
139
+ call_op(:tanh_grad, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
140
+ end
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,99 @@
1
+ module TensorStream
2
+ ## Collection of machine learning related ops
3
+ module NNOps
4
+ def NNOps.included(klass)
5
+ klass.class_eval do
6
+ register_op :apply_gradient_descent do |context, tensor, inputs|
7
+ target_var, learning_rate, delta = inputs
8
+ assign = tensor.inputs[0] || tensor
9
+
10
+ assign.value = process_vector_math_op(tensor, target_var, delta, context, ->(t, u) { t - u * learning_rate })
11
+ assign.value
12
+ end
13
+
14
+ register_op :apply_momentum do |context, tensor, inputs|
15
+ target_var, momentum_var, learning_rate, grad, momentum = inputs
16
+ assign = tensor.inputs[0] || tensor
17
+ assign_acc = tensor.inputs[1]
18
+ assign_acc.value = process_vector_math_op(tensor, momentum_var, grad, context, ->(t, u) { t * momentum + u })
19
+ if tensor.options[:use_nesterov]
20
+ delta = process_vector_math_op(tensor, grad, momentum_var, context, ->(g, acc) { g * learning_rate + acc * momentum * learning_rate })
21
+ assign.value = process_vector_math_op(tensor,target_var, delta, context, ->(t, u) { t - u })
22
+ else
23
+ assign.value = process_vector_math_op(tensor, target_var, momentum_var, context, ->(v, acc) { v - acc * learning_rate })
24
+ end
25
+ assign.value
26
+ end
27
+
28
+ register_op :apply_adam do |context, tensor, inputs|
29
+ target_var, m, v, beta1_power, beta2_power, lr_t, beta1_t, beta2_t, epsilon_t, grad = inputs
30
+ alpha = lr_t * Math.sqrt( 1.0 - beta2_power) / (1.0 - beta1_power)
31
+ assign = tensor.inputs[0]
32
+ assign_m = tensor.inputs[1]
33
+ assign_v = tensor.inputs[2]
34
+
35
+ m_delta = process_vector_math_op(tensor, grad, m, context, ->(g, m_d) { (g - m_d) * (1.0 - beta1_t) })
36
+ assign_m.value = process_vector_math_op(tensor, m, m_delta, context, ->(u_d , v_d) { u_d + v_d })
37
+ assign_v.value = process_vector_math_op(tensor, v, grad, context, ->(u_d , v_d) { u_d + (v_d ** 2 - u_d) * (1.0 - beta2_t)})
38
+ v_delta = process_vector_math_op(tensor, assign_m.value, assign_v.value, context, ->(m_d , v_d) { (m_d * alpha) / (Math.sqrt(v_d) + epsilon_t) })
39
+ assign.value = process_vector_math_op(tensor, target_var, v_delta, context, ->(var_d , delta_d) { var_d - delta_d })
40
+ assign.value
41
+ end
42
+
43
+ register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context, tensor, inputs|
44
+ last_dimen_list = last_axis(inputs[0])
45
+ input_shape = shape_eval(inputs[0])
46
+ rank = input_shape.size - 1
47
+ labels = last_axis(inputs[1])
48
+ func = lambda { |logits, label|
49
+ c = logits.max
50
+ transformed_logits = logits.map { |l| l - c }
51
+ sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
52
+ losses = transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
53
+ probs = transformed_logits.zip(label).map { |x, y| (Math.exp(x) / sum) - y }
54
+ [losses, probs]
55
+ }
56
+
57
+ if input_shape.size == 1
58
+ loss, prob = func.call(last_dimen_list, labels)
59
+ loss = reduce(loss, rank, false)
60
+ TensorStream::Evaluator::OutputGroup.new([loss, prob], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
61
+ else
62
+ losses = []
63
+ backprobs = []
64
+ arr = last_dimen_list.zip(labels).each do |list, label|
65
+ loss, prob = func.call(list, label)
66
+ losses << loss
67
+ backprobs << prob
68
+ end
69
+ reshaped_losses = TensorShape.reshape(losses.flatten, input_shape)
70
+ reshaped_backprops = TensorShape.reshape(backprobs.flatten, input_shape)
71
+ reshaped_losses = reduce(reshaped_losses, rank, false)
72
+ TensorStream::Evaluator::OutputGroup.new([reshaped_losses, reshaped_backprops], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
73
+ end
74
+ end
75
+
76
+ register_op :log_softmax do |_context, _tensor, inputs|
77
+ input_shape = shape_eval(inputs[0])
78
+ last_dimen_list = last_axis(inputs[0])
79
+
80
+ func = lambda { |logits|
81
+ c = logits.max
82
+ transformed_logits = logits.map { |l| l - c }
83
+ sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
84
+ transformed_logits.map { |x| x - Math.log(sum) }
85
+ }
86
+
87
+ if input_shape.size == 1
88
+ func.call(last_dimen_list)
89
+ else
90
+ arr = last_dimen_list.collect do |list|
91
+ func.call(list)
92
+ end
93
+ TensorShape.reshape(arr.flatten, input_shape)
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -2,6 +2,8 @@ require 'tensor_stream/evaluator/operation_helpers/random_gaussian'
2
2
  require 'tensor_stream/evaluator/operation_helpers/array_ops_helper'
3
3
  require 'tensor_stream/evaluator/operation_helpers/math_helper'
4
4
  require 'tensor_stream/evaluator/base_evaluator'
5
+ require 'tensor_stream/evaluator/ruby/math_ops'
6
+ require 'tensor_stream/evaluator/ruby/nn_ops'
5
7
 
6
8
  module TensorStream
7
9
  module Evaluator
@@ -29,6 +31,8 @@ module TensorStream
29
31
  include TensorStream::OpHelper
30
32
  include TensorStream::ArrayOpsHelper
31
33
  include TensorStream::MathHelper
34
+ include TensorStream::MathOps
35
+ include TensorStream::NNOps
32
36
 
33
37
  def run(tensor, execution_context)
34
38
  return tensor.map { |t| run(t, execution_context) } if tensor.is_a?(Array) && !tensor.empty? && tensor[0].is_a?(Tensor)
@@ -243,143 +247,10 @@ module TensorStream
243
247
  Tensor.cast_dtype(input.flatten.size, tensor.options[:out_type])
244
248
  end
245
249
 
246
- register_op %i[neg negate], no_eval: true do |context, tensor, inputs|
247
- call_vector_op(tensor, :negate, inputs[0], nil, context, ->(t, _u) { -t })
248
- end
249
-
250
- register_op :add, no_eval: true do |context, tensor, inputs|
251
- a, b = inputs
252
- call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
253
- end
254
-
255
- register_op :add_n, no_eval: true do |context, tensor, inputs|
256
- if inputs.size == 1
257
- complete_eval(inputs[0], context)
258
- elsif inputs.size > 1
259
-
260
- a = inputs.pop
261
- until inputs.empty?
262
- b = inputs.pop
263
- a = call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
264
- end
265
- a
266
- end
267
- end
268
-
269
- register_op :sub, no_eval: true do |context, tensor, inputs|
270
- a, b = inputs
271
- call_vector_op(tensor, :sub, a, b, context, ->(t, u) { t - u })
272
- end
273
-
274
- register_op %i[floor_mod mod], no_eval: true do |context, tensor, inputs|
275
- a, b = inputs
276
- call_vector_op(tensor, :mod, a, b, context, ->(t, u) { t % u })
277
- end
278
-
279
- register_op %i[floor_div], no_eval: true do |context, tensor, inputs|
280
- a, b = inputs
281
- if fp_type?(tensor.data_type)
282
- call_vector_op(tensor, :div, a, b, context, ->(t, u) { (t / u).to_i.to_f })
283
- else
284
- call_vector_op(tensor, :div, a, b, context, ->(t, u) { t / u })
285
- end
286
- end
287
-
288
- register_op :mul, no_eval: true do |context, tensor, inputs|
289
- a, b = inputs
290
- call_vector_op(tensor, :mul, a, b, context, ->(t, u) { t * u })
291
- end
292
-
293
- register_op :pow, no_eval: true do |context, tensor, inputs|
294
- a, b = inputs
295
- call_vector_op(tensor, :pow, a, b, context, ->(t, u) { t**u })
296
- end
297
-
298
- register_op :squared_difference, no_eval: true do |context, tensor, inputs|
299
- a, b = inputs
300
- call_vector_op(tensor, :squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
301
- end
302
-
303
250
  register_op %i[concat concat_v2] do |_context, tensor, inputs|
304
251
  concat_array(inputs, tensor.options[:axis])
305
252
  end
306
253
 
307
- register_op :round, no_eval: true do |context, _tensor, inputs|
308
- call_op(:round, inputs[0], context, ->(t, _b) { t.round })
309
- end
310
-
311
- register_op :abs, no_eval: true do |context, _tensor, inputs|
312
- call_op(:abs, inputs[0], context, ->(t, _b) { t.abs })
313
- end
314
-
315
- register_op :tanh, no_eval: true do |context, _tensor, inputs|
316
- call_op(:tanh, inputs[0], context, ->(t, _b) { Math.tanh(t) })
317
- end
318
-
319
- register_op :tan, no_eval: true do |context, _tensor, inputs|
320
- call_op(:tan, inputs[0], context, ->(t, _b) { Math.tan(t) })
321
- end
322
-
323
- register_op :atan, no_eval: true do |context, _tensor, inputs|
324
- call_op(:atan, inputs[0], context, ->(t, _b) { Math.atan(t) })
325
- end
326
-
327
- register_op :sec, no_eval: true do |context, _tensor, inputs|
328
- call_op(:sec, inputs[0], context, ->(t, _b) { Math.sec(t) })
329
- end
330
-
331
- register_op :sin, no_eval: true do |context, _tensor, inputs|
332
- call_op(:sin, inputs[0], context, ->(t, _b) { Math.sin(t) })
333
- end
334
-
335
- register_op :asin, no_eval: true do |context, _tensor, inputs|
336
- call_op(:asin, inputs[0], context, ->(t, _b) { Math.asin(t) })
337
- end
338
-
339
- register_op :acos, no_eval: true do |context, _tensor, inputs|
340
- call_op(:acos, inputs[0], context, ->(t, _b) { Math.acos(t) })
341
- end
342
-
343
- register_op :cos, no_eval: true do |context, _tensor, inputs|
344
- call_op(:cos, inputs[0], context, ->(t, _b) { Math.cos(t) })
345
- end
346
-
347
- register_op :log1p, no_eval: true do |context, _tensor, inputs|
348
- call_op(:log1p, inputs[0], context, ->(t, _b) { Math.log(1 + t) })
349
- end
350
-
351
- register_op :log, no_eval: true do |context, _tensor, inputs|
352
- call_op(:log, inputs[0], context, ->(t, _b) { t < 0 ? Float::NAN : Math.log(t) })
353
- end
354
-
355
- register_op :exp, no_eval: true do |context, _tensor, inputs|
356
- call_op(:exp, inputs[0], context, ->(t, _b) { Math.exp(t) })
357
- end
358
-
359
- register_op :sigmoid, no_eval: true do |context, _tensor, inputs|
360
- call_op(:sigmoid, inputs[0], context, ->(t, _b) { sigmoid(t) })
361
- end
362
-
363
- register_op :sqrt, no_eval: true do |context, _tensor, inputs|
364
- call_op(:sqrt, inputs[0], context, ->(t, _b) { Math.sqrt(t) })
365
- end
366
-
367
- register_op :floor, no_eval: true do |context, _tensor, inputs|
368
- call_op(:floor, inputs[0], context, ->(t, _b) { t.floor })
369
- end
370
-
371
- register_op :ceil, no_eval: true do |context, _tensor, inputs|
372
- call_op(:ceil, inputs[0], context, ->(t, _b) { t.ceil })
373
- end
374
-
375
- register_op :square, no_eval: true do |context, _tensor, inputs|
376
- call_op(:square, inputs[0], context, ->(t, _b) { t * t })
377
- end
378
-
379
- register_op :reciprocal, no_eval: true do |context, _tensor, inputs|
380
- call_op(:reciprocal, inputs[0], context, ->(t, _b) { 1 / t })
381
- end
382
-
383
254
  register_op :stop_gradient, no_eval: true do |_context, _tensor, inputs|
384
255
  inputs[0]
385
256
  end
@@ -517,14 +388,6 @@ module TensorStream
517
388
  end
518
389
 
519
390
  register_op :sum, noop: true do |context, tensor, _inputs|
520
- # axis = complete_eval(tensor.inputs[1], context)
521
- # # fast path
522
- # if axis.nil? && !tensor.options[:keepdims]
523
- # arr = complete_eval(tensor.inputs[0], context)
524
- # next arr unless arr.is_a?(Array)
525
- # next arr.flatten.reduce(:+)
526
- # end
527
-
528
391
  func = lambda do |arr|
529
392
  reduced_val = arr[0]
530
393
  arr[1..arr.size].each do |v|
@@ -537,14 +400,6 @@ module TensorStream
537
400
  end
538
401
 
539
402
  register_op :prod, noop: true do |context, tensor, _inputs|
540
- # axis = complete_eval(tensor.inputs[1], context)
541
- # # fast path
542
- # if axis.nil? && !tensor.options[:keepdims]
543
- # arr = complete_eval(tensor.inputs[0], context)
544
- # next arr unless arr.is_a?(Array)
545
- # next arr.flatten.reduce(:*)
546
- # end
547
-
548
403
  c = fp_type?(tensor.data_type) ? 1.0 : 1
549
404
  func = lambda do |arr|
550
405
  return c if arr.nil?
@@ -577,10 +432,6 @@ module TensorStream
577
432
  r
578
433
  end
579
434
 
580
- register_op :tanh_grad, no_eval: true do |context, _tensor, inputs|
581
- call_op(:tanh_grad, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
582
- end
583
-
584
435
  register_op :transpose do |_context, tensor, inputs|
585
436
  shape = shape_eval(inputs[0])
586
437
  rank = get_rank(inputs[0])
@@ -774,28 +625,6 @@ module TensorStream
774
625
  call_vector_op(tensor, :min, inputs[0], inputs[1], context, ->(t, u) { [t, u].min })
775
626
  end
776
627
 
777
- register_op :apply_gradient_descent do |context, tensor, inputs|
778
- target_var, learning_rate, delta = inputs
779
- assign = tensor.inputs[0] || tensor
780
-
781
- assign.value = process_vector_math_op(tensor, target_var, delta, context, ->(t, u) { t - u * learning_rate })
782
- assign.value
783
- end
784
-
785
- register_op :apply_momentum do |context, tensor, inputs|
786
- target_var, momentum_var, learning_rate, grad, momentum = inputs
787
- assign = tensor.inputs[0] || tensor
788
- assign_acc = tensor.inputs[1]
789
- assign_acc.value = process_vector_math_op(tensor, momentum_var, grad, context, ->(t, u) { t * momentum + u })
790
- if tensor.options[:use_nesterov]
791
- delta = process_vector_math_op(tensor, grad, momentum_var, context, ->(g, acc) { g * learning_rate + acc * momentum * learning_rate })
792
- assign.value = process_vector_math_op(tensor,target_var, delta, context, ->(t, u) { t - u })
793
- else
794
- assign.value = process_vector_math_op(tensor, target_var, momentum_var, context, ->(v, acc) { v - acc * learning_rate })
795
- end
796
- assign.value
797
- end
798
-
799
628
  register_op :broadcast_gradient_args do |_context, tensor, inputs|
800
629
  rx, ry = get_broadcast_gradient_args(inputs[0], inputs[1])
801
630
  OutputGroup.new([rx, ry], tensor.inputs.map(&:data_type))
@@ -812,7 +641,8 @@ module TensorStream
812
641
  end
813
642
 
814
643
  register_op :flow_group, noop: true do |context, tensor, inputs|
815
- inputs.collect { |input| global_eval(tensor, input, context) }
644
+ inputs.each { |input| global_eval(tensor, input, context) }
645
+ nil # no output
816
646
  end
817
647
 
818
648
  register_op :softmax do |_context, _tensor, inputs|
@@ -856,83 +686,6 @@ module TensorStream
856
686
  end
857
687
  end
858
688
 
859
- register_op :log_softmax do |_context, _tensor, inputs|
860
- input_shape = shape_eval(inputs[0])
861
- last_dimen_list = last_axis(inputs[0])
862
-
863
- func = lambda { |logits|
864
- c = logits.max
865
- transformed_logits = logits.map { |l| l - c }
866
- sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
867
- transformed_logits.map { |x| x - Math.log(sum) }
868
- }
869
-
870
- if input_shape.size == 1
871
- func.call(last_dimen_list)
872
- else
873
- arr = last_dimen_list.collect do |list|
874
- func.call(list)
875
- end
876
- TensorShape.reshape(arr.flatten, input_shape)
877
- end
878
- end
879
-
880
- register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context, tensor, inputs|
881
- last_dimen_list = last_axis(inputs[0])
882
- input_shape = shape_eval(inputs[0])
883
- rank = input_shape.size - 1
884
- labels = last_axis(inputs[1])
885
- func = lambda { |logits, label|
886
- c = logits.max
887
- transformed_logits = logits.map { |l| l - c }
888
- sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
889
- losses = transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
890
- probs = transformed_logits.zip(label).map { |x, y| (Math.exp(x) / sum) - y }
891
- [losses, probs]
892
- }
893
-
894
- if input_shape.size == 1
895
- loss, prob = func.call(last_dimen_list, labels)
896
- loss = reduce(loss, rank, false)
897
- OutputGroup.new([loss, prob], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
898
- else
899
- losses = []
900
- backprobs = []
901
- arr = last_dimen_list.zip(labels).each do |list, label|
902
- loss, prob = func.call(list, label)
903
- losses << loss
904
- backprobs << prob
905
- end
906
- reshaped_losses = TensorShape.reshape(losses.flatten, input_shape)
907
- reshaped_backprops = TensorShape.reshape(backprobs.flatten, input_shape)
908
- reshaped_losses = reduce(reshaped_losses, rank, false)
909
- OutputGroup.new([reshaped_losses, reshaped_backprops], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
910
- end
911
- end
912
-
913
- register_op :softmax_cross_entropy_with_logits_v2_grad do |_context, _tensor, inputs|
914
- last_dimen_list = last_axis(inputs[0])
915
- labels = last_axis(inputs[1])
916
- passed_grads = last_axis(inputs[2])
917
- input_shape = shape_eval(inputs[0])
918
-
919
- func = lambda { |logits, label, grad|
920
- c = logits.max
921
- transformed_logits = logits.map { |l| Math.exp(l - c) }
922
- e_sum = transformed_logits.reduce(:+)
923
- transformed_logits.zip(label).zip(grad).map { |(x, y), g| (x / e_sum) * g - y }
924
- }
925
-
926
- if input_shape.size == 1
927
- func.call(last_dimen_list, labels, passed_grads)
928
- else
929
- arr = last_dimen_list.zip(labels).zip(passed_grads).collect do |(list, label), passed_grad|
930
- func.call(list, label, passed_grad)
931
- end
932
- TensorShape.reshape(arr.flatten, input_shape)
933
- end
934
- end
935
-
936
689
  register_op :check_numerics do |context, tensor, inputs|
937
690
  message = tensor.options[:message]
938
691
  f = lambda { |t, _b|