tensor_stream 0.8.0 → 0.8.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: f84c2b9852fcf4931c47c0130b67497a50a87b0f
4
- data.tar.gz: 524e1105da4e06e3472cbcfa0e6f764ae4512d37
2
+ SHA256:
3
+ metadata.gz: a3c7d0a810a79ceedc0237379b105d7a9b598ba2513ef2d59ba3cec78d7b0da0
4
+ data.tar.gz: 7c0d90b27e548b72a86e88e7181f3d3b131a5fa3c6800000c743dd6e47d47b3b
5
5
  SHA512:
6
- metadata.gz: 420e2675ab67d4c8462534bdf8c703671656f7852d984579e22ee57f1425dd5740fcb64a1e52363bf337cd7c691d87a75c76bd868b13c8a7f06d78e0eb00aa73
7
- data.tar.gz: 24fe1022741883d46cdd5af51309da33d421d72874f0cc84bf2e0ed14a62602f1830c6060bd86e42359b7962b4a57727c9a48ce13d5950d5ba02f6a9cdfd719f
6
+ metadata.gz: a7a8a5607883d868da3ceaa9f870ac5c6d6809d45992a9ae0e4f26d5648bda2c4a26ea3abec506abd92362dcf3b63935b3f6acbcc70b605600307313f8c69f49
7
+ data.tar.gz: db8917bee53f91e1017b5fdb8b9ece8b50a1096d249cee40eed8c335f597f32e5ce81056230b8ce1b9acff4df728497fa13e896212cbdaccc023dbdb7ed3591e
data/CHANGELOG.md CHANGED
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
4
4
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
5
5
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
6
6
 
7
+
8
+ ## [0.8.1] - 2018-08-30
9
+ - [TRAINING] Added AdamOptimizer
10
+
7
11
  ## [0.8.0] - 2018-08-29
8
12
  ### Added
9
13
  - [TRAINING] Added new supported optimizer, MomentumOptimizer loosely based on tensorflow's implementation (with nesterov support)
data/README.md CHANGED
@@ -17,7 +17,20 @@ The goal of this gem is to have a high performance machine learning and compute
17
17
  - eager execution (experimental)
18
18
  - (08-08-2018) Load pbtext files from tensorflow (Graph.parse_from_string)
19
19
 
20
- Since this is a pure ruby implementation for now, performance is not there yet. However it should be a good enough environment to learn about tensorflow and experiment with some models.
20
+ ## Compatibility
21
+
22
+ TensorStream comes with a pure ruby and OpenCL implementation out of the box. The pure ruby implementation
23
+ is known to work with most ruby implementations including TruffleRuby, JRuby as well as jit enabled versions of mri (ruby-2.6.0).
24
+
25
+ OpenCL is supported only on mri implementations of ruby. This can be enabled by including the OpenCL evaluator (Make sure you have OpenCL drivers installed correctly on your system):
26
+
27
+ ```ruby
28
+ require 'tensor_stream/evaluator/opencl/opencl_evaluator'
29
+ ```
30
+
31
+ OpenCL is basically a requirement for deep learning and image processing tasks as the ruby implementation is too slow even with jit speedups using latest ruby implementations.
32
+
33
+ OpenCL kernels used by tensorstream can be found at tensor_stream/lib/evaluator/opencl/kernels. These are non specific and should work with any device that supports OpenCL including intel GPUs and CPUs, as well as GPUs from Nvidia and AMD.
21
34
 
22
35
  ## Installation
23
36
 
@@ -72,6 +85,8 @@ pred = X * W + b
72
85
  # Mean squared error
73
86
  cost = ((pred - Y) ** 2).reduce(:+) / ( 2 * n_samples)
74
87
 
88
+ # optimizer = TensorStream::Train::MomentumOptimizer.new(0.01, 0.5, use_nesterov: true).minimize(cost)
89
+ # optimizer = TensorStream::Train::AdamOptimizer.new.minimize(cost)
75
90
  optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate).minimize(cost)
76
91
 
77
92
  # Initialize the variables (i.e. assign their default value)
@@ -338,7 +353,7 @@ ruby 2.4
338
353
  $ ruby -v
339
354
  ruby 2.4.0p0 (2016-12-24 revision 57164) [x86_64-linux]
340
355
  $ ruby samples/linear_regression.rb
341
- 495 seconds 1000 epochs
356
+ 495 seconds 10000 epochs
342
357
  ```
343
358
 
344
359
  ruby 2.6.0-preview2
@@ -383,8 +398,6 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
383
398
 
384
399
  Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/tensor_stream. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
385
400
 
386
-
387
401
  ## License
388
402
 
389
- The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
390
-
403
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
@@ -0,0 +1,23 @@
1
+ % c_dtype = dtype_to_c_type(dtype)
2
+ // same dimension add floating point op
3
+ __kernel void apply_adam_<%= dtype %>(const int M, const int N,
4
+ __global const <%= c_dtype %> *grad,
5
+ __global const <%= c_dtype %> *learning_rate,
6
+ __global const <%= c_dtype %> *beta1_power,
7
+ __global const <%= c_dtype %> *beta2_power,
8
+ __global const <%= c_dtype %> *beta1,
9
+ __global const <%= c_dtype %> *beta2,
10
+ __global const <%= c_dtype %> *epsilon,
11
+ __global <%= c_dtype %> *momentum,
12
+ __global <%= c_dtype %> *output, __global <%= c_dtype %> *v) {
13
+ // Get the index of the current element to be processed
14
+ const int globalRow = get_global_id(0); // Row ID of C (0..M)
15
+ const int globalCol = get_global_id(1); // Col ID of C (0..N)
16
+ const int index = globalRow * N + globalCol;
17
+
18
+ <%= c_dtype %> alpha = learning_rate[0] * sqrt(1.0 - beta2_power[0]) / (1.0 - beta1_power[0]);
19
+
20
+ momentum[index] += (grad[index] - momentum[index]) * (1.0 - beta1[0]);
21
+ v[index] += (grad[index] * grad[index] - v[index]) * (1.0 - beta2[0]);
22
+ output[index] -= (momentum[index] * alpha) / ( sqrt(v[index]) + epsilon[0] );
23
+ }
@@ -107,9 +107,8 @@ module TensorStream
107
107
  end
108
108
  end
109
109
 
110
- def complete_eval(tensor, context)
110
+ def enqueue_buffer_read(tensor, context)
111
111
  buffer = _run(tensor, context)
112
-
113
112
  if buffer.is_a?(Array)
114
113
  buffer = buffer.collect do |b|
115
114
  next b if b.buffer.size.zero?
@@ -122,7 +121,12 @@ module TensorStream
122
121
  return [] if buffer.buffer.nil?
123
122
  return buffer if buffer.buffer.size.zero?
124
123
  _opencl_queue.enqueue_read_buffer(buffer.cl_buffer, buffer.buffer, event_wait_list: build_event_wait_list([buffer]))
124
+ buffer
125
125
  end
126
+ end
127
+
128
+ def complete_eval(tensor, context)
129
+ buffer = enqueue_buffer_read(tensor, context)
126
130
  _opencl_queue.finish
127
131
  buffer
128
132
  end
@@ -339,6 +343,48 @@ module TensorStream
339
343
  learning_rate.cl_buffer, momentum.cl_buffer, output_buffer.cl_buffer,
340
344
  assign_acc.buffer.cl_buffer, event_wait_list: event_wait_list)
341
345
  output_buffer.op = event
346
+ assign_acc.buffer.op = event
347
+ output_buffer
348
+ end
349
+
350
+ # Adam optimization algorithm
351
+ register_op :apply_adam do |_context, tensor, inputs|
352
+ _target_var, _m, _v, beta1_power, beta2_power, lr_t, beta1_t, beta2_t, epsilon_t, grad = inputs
353
+
354
+ assign = tensor.inputs[0] || tensor
355
+ assign_m = tensor.inputs[1]
356
+ assign_v = tensor.inputs[2]
357
+
358
+ # mark variable buffers as dirty
359
+ assign.buffer.dirty = true # force buffer copy when variable is read externally
360
+ assign_m.buffer.dirty = true # force buffer copy when variable is read externally
361
+ assign_v.buffer.dirty = true # force buffer copy when variable is read externally
362
+
363
+ output_buffer = assign.buffer
364
+
365
+ m, n = output_buffer.shape
366
+ work_group = [m || 1, n || 1]
367
+ cl_m = OpenCL::Int1.new(m || 1)
368
+ cl_n = OpenCL::Int1.new(n || 1)
369
+
370
+ event_wait_list = build_event_wait_list(inputs)
371
+ method_call = :"apply_adam_#{output_buffer.data_type}"
372
+ event = _cl_program("apply_adam", dtype: output_buffer.data_type)
373
+ .send(method_call, _opencl_queue, work_group, cl_m, cl_n,
374
+ grad.cl_buffer,
375
+ lr_t.cl_buffer,
376
+ beta1_power.cl_buffer,
377
+ beta2_power.cl_buffer,
378
+ beta1_t.cl_buffer,
379
+ beta2_t.cl_buffer,
380
+ epsilon_t.cl_buffer,
381
+ assign_m.buffer.cl_buffer,
382
+ assign.buffer.cl_buffer,
383
+ assign_v.buffer.cl_buffer,
384
+ event_wait_list: event_wait_list)
385
+ output_buffer.op = event
386
+ assign_m.buffer.op = event
387
+ assign_v.buffer.op = event
342
388
  output_buffer
343
389
  end
344
390
 
@@ -713,8 +759,9 @@ module TensorStream
713
759
  convert_to_opencl(arr.buffer, shape, data_type: arr.data_type, name: tensor.name)
714
760
  end
715
761
 
716
- register_op :flow_group do |_context, _tensor, inputs|
717
- inputs
762
+ register_op :flow_group do |context, _tensor, inputs|
763
+ _opencl_queue.finish
764
+ nil
718
765
  end
719
766
 
720
767
  register_op :size do |_context, tensor, inputs|
@@ -0,0 +1,144 @@
1
+ module TensorStream
2
+ module MathOps
3
+ def MathOps.included(klass)
4
+ klass.class_eval do
5
+ register_op :tanh, no_eval: true do |context, _tensor, inputs|
6
+ call_op(:tanh, inputs[0], context, ->(t, _b) { Math.tanh(t) })
7
+ end
8
+
9
+ register_op :tan, no_eval: true do |context, _tensor, inputs|
10
+ call_op(:tan, inputs[0], context, ->(t, _b) { Math.tan(t) })
11
+ end
12
+
13
+ register_op :atan, no_eval: true do |context, _tensor, inputs|
14
+ call_op(:atan, inputs[0], context, ->(t, _b) { Math.atan(t) })
15
+ end
16
+
17
+ register_op :sec, no_eval: true do |context, _tensor, inputs|
18
+ call_op(:sec, inputs[0], context, ->(t, _b) { Math.sec(t) })
19
+ end
20
+
21
+ register_op :sin, no_eval: true do |context, _tensor, inputs|
22
+ call_op(:sin, inputs[0], context, ->(t, _b) { Math.sin(t) })
23
+ end
24
+
25
+ register_op :add, no_eval: true do |context, tensor, inputs|
26
+ a, b = inputs
27
+ call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
28
+ end
29
+
30
+ register_op :add_n, no_eval: true do |context, tensor, inputs|
31
+ if inputs.size == 1
32
+ complete_eval(inputs[0], context)
33
+ elsif inputs.size > 1
34
+
35
+ a = inputs.pop
36
+ until inputs.empty?
37
+ b = inputs.pop
38
+ a = call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
39
+ end
40
+ a
41
+ end
42
+ end
43
+
44
+ register_op :sub, no_eval: true do |context, tensor, inputs|
45
+ a, b = inputs
46
+ call_vector_op(tensor, :sub, a, b, context, ->(t, u) { t - u })
47
+ end
48
+
49
+ register_op %i[floor_mod mod], no_eval: true do |context, tensor, inputs|
50
+ a, b = inputs
51
+ call_vector_op(tensor, :mod, a, b, context, ->(t, u) { t % u })
52
+ end
53
+
54
+ register_op %i[floor_div], no_eval: true do |context, tensor, inputs|
55
+ a, b = inputs
56
+ if fp_type?(tensor.data_type)
57
+ call_vector_op(tensor, :div, a, b, context, ->(t, u) { (t / u).to_i.to_f })
58
+ else
59
+ call_vector_op(tensor, :div, a, b, context, ->(t, u) { t / u })
60
+ end
61
+ end
62
+
63
+ register_op :mul, no_eval: true do |context, tensor, inputs|
64
+ a, b = inputs
65
+ call_vector_op(tensor, :mul, a, b, context, ->(t, u) { t * u })
66
+ end
67
+
68
+ register_op :pow, no_eval: true do |context, tensor, inputs|
69
+ a, b = inputs
70
+ call_vector_op(tensor, :pow, a, b, context, ->(t, u) { t**u })
71
+ end
72
+
73
+ register_op :squared_difference, no_eval: true do |context, tensor, inputs|
74
+ a, b = inputs
75
+ call_vector_op(tensor, :squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
76
+ end
77
+
78
+ register_op :round, no_eval: true do |context, _tensor, inputs|
79
+ call_op(:round, inputs[0], context, ->(t, _b) { t.round })
80
+ end
81
+
82
+ register_op :abs, no_eval: true do |context, _tensor, inputs|
83
+ call_op(:abs, inputs[0], context, ->(t, _b) { t.abs })
84
+ end
85
+
86
+ register_op :asin, no_eval: true do |context, _tensor, inputs|
87
+ call_op(:asin, inputs[0], context, ->(t, _b) { Math.asin(t) })
88
+ end
89
+
90
+ register_op :acos, no_eval: true do |context, _tensor, inputs|
91
+ call_op(:acos, inputs[0], context, ->(t, _b) { Math.acos(t) })
92
+ end
93
+
94
+ register_op :cos, no_eval: true do |context, _tensor, inputs|
95
+ call_op(:cos, inputs[0], context, ->(t, _b) { Math.cos(t) })
96
+ end
97
+
98
+ register_op :log1p, no_eval: true do |context, _tensor, inputs|
99
+ call_op(:log1p, inputs[0], context, ->(t, _b) { Math.log(1 + t) })
100
+ end
101
+
102
+ register_op :log, no_eval: true do |context, _tensor, inputs|
103
+ call_op(:log, inputs[0], context, ->(t, _b) { t < 0 ? Float::NAN : Math.log(t) })
104
+ end
105
+
106
+ register_op :exp, no_eval: true do |context, _tensor, inputs|
107
+ call_op(:exp, inputs[0], context, ->(t, _b) { Math.exp(t) })
108
+ end
109
+
110
+ register_op :sigmoid, no_eval: true do |context, _tensor, inputs|
111
+ call_op(:sigmoid, inputs[0], context, ->(t, _b) { sigmoid(t) })
112
+ end
113
+
114
+ register_op :sqrt, no_eval: true do |context, _tensor, inputs|
115
+ call_op(:sqrt, inputs[0], context, ->(t, _b) { Math.sqrt(t) })
116
+ end
117
+
118
+ register_op :floor, no_eval: true do |context, _tensor, inputs|
119
+ call_op(:floor, inputs[0], context, ->(t, _b) { t.floor })
120
+ end
121
+
122
+ register_op :ceil, no_eval: true do |context, _tensor, inputs|
123
+ call_op(:ceil, inputs[0], context, ->(t, _b) { t.ceil })
124
+ end
125
+
126
+ register_op :square, no_eval: true do |context, _tensor, inputs|
127
+ call_op(:square, inputs[0], context, ->(t, _b) { t * t })
128
+ end
129
+
130
+ register_op :reciprocal, no_eval: true do |context, _tensor, inputs|
131
+ call_op(:reciprocal, inputs[0], context, ->(t, _b) { 1 / t })
132
+ end
133
+
134
+ register_op %i[neg negate], no_eval: true do |context, tensor, inputs|
135
+ call_vector_op(tensor, :negate, inputs[0], nil, context, ->(t, _u) { -t })
136
+ end
137
+
138
+ register_op :tanh_grad, no_eval: true do |context, _tensor, inputs|
139
+ call_op(:tanh_grad, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
140
+ end
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,99 @@
1
+ module TensorStream
2
+ ## Collection of machine learning related ops
3
+ module NNOps
4
+ def NNOps.included(klass)
5
+ klass.class_eval do
6
+ register_op :apply_gradient_descent do |context, tensor, inputs|
7
+ target_var, learning_rate, delta = inputs
8
+ assign = tensor.inputs[0] || tensor
9
+
10
+ assign.value = process_vector_math_op(tensor, target_var, delta, context, ->(t, u) { t - u * learning_rate })
11
+ assign.value
12
+ end
13
+
14
+ register_op :apply_momentum do |context, tensor, inputs|
15
+ target_var, momentum_var, learning_rate, grad, momentum = inputs
16
+ assign = tensor.inputs[0] || tensor
17
+ assign_acc = tensor.inputs[1]
18
+ assign_acc.value = process_vector_math_op(tensor, momentum_var, grad, context, ->(t, u) { t * momentum + u })
19
+ if tensor.options[:use_nesterov]
20
+ delta = process_vector_math_op(tensor, grad, momentum_var, context, ->(g, acc) { g * learning_rate + acc * momentum * learning_rate })
21
+ assign.value = process_vector_math_op(tensor,target_var, delta, context, ->(t, u) { t - u })
22
+ else
23
+ assign.value = process_vector_math_op(tensor, target_var, momentum_var, context, ->(v, acc) { v - acc * learning_rate })
24
+ end
25
+ assign.value
26
+ end
27
+
28
+ register_op :apply_adam do |context, tensor, inputs|
29
+ target_var, m, v, beta1_power, beta2_power, lr_t, beta1_t, beta2_t, epsilon_t, grad = inputs
30
+ alpha = lr_t * Math.sqrt( 1.0 - beta2_power) / (1.0 - beta1_power)
31
+ assign = tensor.inputs[0]
32
+ assign_m = tensor.inputs[1]
33
+ assign_v = tensor.inputs[2]
34
+
35
+ m_delta = process_vector_math_op(tensor, grad, m, context, ->(g, m_d) { (g - m_d) * (1.0 - beta1_t) })
36
+ assign_m.value = process_vector_math_op(tensor, m, m_delta, context, ->(u_d , v_d) { u_d + v_d })
37
+ assign_v.value = process_vector_math_op(tensor, v, grad, context, ->(u_d , v_d) { u_d + (v_d ** 2 - u_d) * (1.0 - beta2_t)})
38
+ v_delta = process_vector_math_op(tensor, assign_m.value, assign_v.value, context, ->(m_d , v_d) { (m_d * alpha) / (Math.sqrt(v_d) + epsilon_t) })
39
+ assign.value = process_vector_math_op(tensor, target_var, v_delta, context, ->(var_d , delta_d) { var_d - delta_d })
40
+ assign.value
41
+ end
42
+
43
+ register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context, tensor, inputs|
44
+ last_dimen_list = last_axis(inputs[0])
45
+ input_shape = shape_eval(inputs[0])
46
+ rank = input_shape.size - 1
47
+ labels = last_axis(inputs[1])
48
+ func = lambda { |logits, label|
49
+ c = logits.max
50
+ transformed_logits = logits.map { |l| l - c }
51
+ sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
52
+ losses = transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
53
+ probs = transformed_logits.zip(label).map { |x, y| (Math.exp(x) / sum) - y }
54
+ [losses, probs]
55
+ }
56
+
57
+ if input_shape.size == 1
58
+ loss, prob = func.call(last_dimen_list, labels)
59
+ loss = reduce(loss, rank, false)
60
+ TensorStream::Evaluator::OutputGroup.new([loss, prob], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
61
+ else
62
+ losses = []
63
+ backprobs = []
64
+ arr = last_dimen_list.zip(labels).each do |list, label|
65
+ loss, prob = func.call(list, label)
66
+ losses << loss
67
+ backprobs << prob
68
+ end
69
+ reshaped_losses = TensorShape.reshape(losses.flatten, input_shape)
70
+ reshaped_backprops = TensorShape.reshape(backprobs.flatten, input_shape)
71
+ reshaped_losses = reduce(reshaped_losses, rank, false)
72
+ TensorStream::Evaluator::OutputGroup.new([reshaped_losses, reshaped_backprops], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
73
+ end
74
+ end
75
+
76
+ register_op :log_softmax do |_context, _tensor, inputs|
77
+ input_shape = shape_eval(inputs[0])
78
+ last_dimen_list = last_axis(inputs[0])
79
+
80
+ func = lambda { |logits|
81
+ c = logits.max
82
+ transformed_logits = logits.map { |l| l - c }
83
+ sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
84
+ transformed_logits.map { |x| x - Math.log(sum) }
85
+ }
86
+
87
+ if input_shape.size == 1
88
+ func.call(last_dimen_list)
89
+ else
90
+ arr = last_dimen_list.collect do |list|
91
+ func.call(list)
92
+ end
93
+ TensorShape.reshape(arr.flatten, input_shape)
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -2,6 +2,8 @@ require 'tensor_stream/evaluator/operation_helpers/random_gaussian'
2
2
  require 'tensor_stream/evaluator/operation_helpers/array_ops_helper'
3
3
  require 'tensor_stream/evaluator/operation_helpers/math_helper'
4
4
  require 'tensor_stream/evaluator/base_evaluator'
5
+ require 'tensor_stream/evaluator/ruby/math_ops'
6
+ require 'tensor_stream/evaluator/ruby/nn_ops'
5
7
 
6
8
  module TensorStream
7
9
  module Evaluator
@@ -29,6 +31,8 @@ module TensorStream
29
31
  include TensorStream::OpHelper
30
32
  include TensorStream::ArrayOpsHelper
31
33
  include TensorStream::MathHelper
34
+ include TensorStream::MathOps
35
+ include TensorStream::NNOps
32
36
 
33
37
  def run(tensor, execution_context)
34
38
  return tensor.map { |t| run(t, execution_context) } if tensor.is_a?(Array) && !tensor.empty? && tensor[0].is_a?(Tensor)
@@ -243,143 +247,10 @@ module TensorStream
243
247
  Tensor.cast_dtype(input.flatten.size, tensor.options[:out_type])
244
248
  end
245
249
 
246
- register_op %i[neg negate], no_eval: true do |context, tensor, inputs|
247
- call_vector_op(tensor, :negate, inputs[0], nil, context, ->(t, _u) { -t })
248
- end
249
-
250
- register_op :add, no_eval: true do |context, tensor, inputs|
251
- a, b = inputs
252
- call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
253
- end
254
-
255
- register_op :add_n, no_eval: true do |context, tensor, inputs|
256
- if inputs.size == 1
257
- complete_eval(inputs[0], context)
258
- elsif inputs.size > 1
259
-
260
- a = inputs.pop
261
- until inputs.empty?
262
- b = inputs.pop
263
- a = call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
264
- end
265
- a
266
- end
267
- end
268
-
269
- register_op :sub, no_eval: true do |context, tensor, inputs|
270
- a, b = inputs
271
- call_vector_op(tensor, :sub, a, b, context, ->(t, u) { t - u })
272
- end
273
-
274
- register_op %i[floor_mod mod], no_eval: true do |context, tensor, inputs|
275
- a, b = inputs
276
- call_vector_op(tensor, :mod, a, b, context, ->(t, u) { t % u })
277
- end
278
-
279
- register_op %i[floor_div], no_eval: true do |context, tensor, inputs|
280
- a, b = inputs
281
- if fp_type?(tensor.data_type)
282
- call_vector_op(tensor, :div, a, b, context, ->(t, u) { (t / u).to_i.to_f })
283
- else
284
- call_vector_op(tensor, :div, a, b, context, ->(t, u) { t / u })
285
- end
286
- end
287
-
288
- register_op :mul, no_eval: true do |context, tensor, inputs|
289
- a, b = inputs
290
- call_vector_op(tensor, :mul, a, b, context, ->(t, u) { t * u })
291
- end
292
-
293
- register_op :pow, no_eval: true do |context, tensor, inputs|
294
- a, b = inputs
295
- call_vector_op(tensor, :pow, a, b, context, ->(t, u) { t**u })
296
- end
297
-
298
- register_op :squared_difference, no_eval: true do |context, tensor, inputs|
299
- a, b = inputs
300
- call_vector_op(tensor, :squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
301
- end
302
-
303
250
  register_op %i[concat concat_v2] do |_context, tensor, inputs|
304
251
  concat_array(inputs, tensor.options[:axis])
305
252
  end
306
253
 
307
- register_op :round, no_eval: true do |context, _tensor, inputs|
308
- call_op(:round, inputs[0], context, ->(t, _b) { t.round })
309
- end
310
-
311
- register_op :abs, no_eval: true do |context, _tensor, inputs|
312
- call_op(:abs, inputs[0], context, ->(t, _b) { t.abs })
313
- end
314
-
315
- register_op :tanh, no_eval: true do |context, _tensor, inputs|
316
- call_op(:tanh, inputs[0], context, ->(t, _b) { Math.tanh(t) })
317
- end
318
-
319
- register_op :tan, no_eval: true do |context, _tensor, inputs|
320
- call_op(:tan, inputs[0], context, ->(t, _b) { Math.tan(t) })
321
- end
322
-
323
- register_op :atan, no_eval: true do |context, _tensor, inputs|
324
- call_op(:atan, inputs[0], context, ->(t, _b) { Math.atan(t) })
325
- end
326
-
327
- register_op :sec, no_eval: true do |context, _tensor, inputs|
328
- call_op(:sec, inputs[0], context, ->(t, _b) { Math.sec(t) })
329
- end
330
-
331
- register_op :sin, no_eval: true do |context, _tensor, inputs|
332
- call_op(:sin, inputs[0], context, ->(t, _b) { Math.sin(t) })
333
- end
334
-
335
- register_op :asin, no_eval: true do |context, _tensor, inputs|
336
- call_op(:asin, inputs[0], context, ->(t, _b) { Math.asin(t) })
337
- end
338
-
339
- register_op :acos, no_eval: true do |context, _tensor, inputs|
340
- call_op(:acos, inputs[0], context, ->(t, _b) { Math.acos(t) })
341
- end
342
-
343
- register_op :cos, no_eval: true do |context, _tensor, inputs|
344
- call_op(:cos, inputs[0], context, ->(t, _b) { Math.cos(t) })
345
- end
346
-
347
- register_op :log1p, no_eval: true do |context, _tensor, inputs|
348
- call_op(:log1p, inputs[0], context, ->(t, _b) { Math.log(1 + t) })
349
- end
350
-
351
- register_op :log, no_eval: true do |context, _tensor, inputs|
352
- call_op(:log, inputs[0], context, ->(t, _b) { t < 0 ? Float::NAN : Math.log(t) })
353
- end
354
-
355
- register_op :exp, no_eval: true do |context, _tensor, inputs|
356
- call_op(:exp, inputs[0], context, ->(t, _b) { Math.exp(t) })
357
- end
358
-
359
- register_op :sigmoid, no_eval: true do |context, _tensor, inputs|
360
- call_op(:sigmoid, inputs[0], context, ->(t, _b) { sigmoid(t) })
361
- end
362
-
363
- register_op :sqrt, no_eval: true do |context, _tensor, inputs|
364
- call_op(:sqrt, inputs[0], context, ->(t, _b) { Math.sqrt(t) })
365
- end
366
-
367
- register_op :floor, no_eval: true do |context, _tensor, inputs|
368
- call_op(:floor, inputs[0], context, ->(t, _b) { t.floor })
369
- end
370
-
371
- register_op :ceil, no_eval: true do |context, _tensor, inputs|
372
- call_op(:ceil, inputs[0], context, ->(t, _b) { t.ceil })
373
- end
374
-
375
- register_op :square, no_eval: true do |context, _tensor, inputs|
376
- call_op(:square, inputs[0], context, ->(t, _b) { t * t })
377
- end
378
-
379
- register_op :reciprocal, no_eval: true do |context, _tensor, inputs|
380
- call_op(:reciprocal, inputs[0], context, ->(t, _b) { 1 / t })
381
- end
382
-
383
254
  register_op :stop_gradient, no_eval: true do |_context, _tensor, inputs|
384
255
  inputs[0]
385
256
  end
@@ -517,14 +388,6 @@ module TensorStream
517
388
  end
518
389
 
519
390
  register_op :sum, noop: true do |context, tensor, _inputs|
520
- # axis = complete_eval(tensor.inputs[1], context)
521
- # # fast path
522
- # if axis.nil? && !tensor.options[:keepdims]
523
- # arr = complete_eval(tensor.inputs[0], context)
524
- # next arr unless arr.is_a?(Array)
525
- # next arr.flatten.reduce(:+)
526
- # end
527
-
528
391
  func = lambda do |arr|
529
392
  reduced_val = arr[0]
530
393
  arr[1..arr.size].each do |v|
@@ -537,14 +400,6 @@ module TensorStream
537
400
  end
538
401
 
539
402
  register_op :prod, noop: true do |context, tensor, _inputs|
540
- # axis = complete_eval(tensor.inputs[1], context)
541
- # # fast path
542
- # if axis.nil? && !tensor.options[:keepdims]
543
- # arr = complete_eval(tensor.inputs[0], context)
544
- # next arr unless arr.is_a?(Array)
545
- # next arr.flatten.reduce(:*)
546
- # end
547
-
548
403
  c = fp_type?(tensor.data_type) ? 1.0 : 1
549
404
  func = lambda do |arr|
550
405
  return c if arr.nil?
@@ -577,10 +432,6 @@ module TensorStream
577
432
  r
578
433
  end
579
434
 
580
- register_op :tanh_grad, no_eval: true do |context, _tensor, inputs|
581
- call_op(:tanh_grad, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
582
- end
583
-
584
435
  register_op :transpose do |_context, tensor, inputs|
585
436
  shape = shape_eval(inputs[0])
586
437
  rank = get_rank(inputs[0])
@@ -774,28 +625,6 @@ module TensorStream
774
625
  call_vector_op(tensor, :min, inputs[0], inputs[1], context, ->(t, u) { [t, u].min })
775
626
  end
776
627
 
777
- register_op :apply_gradient_descent do |context, tensor, inputs|
778
- target_var, learning_rate, delta = inputs
779
- assign = tensor.inputs[0] || tensor
780
-
781
- assign.value = process_vector_math_op(tensor, target_var, delta, context, ->(t, u) { t - u * learning_rate })
782
- assign.value
783
- end
784
-
785
- register_op :apply_momentum do |context, tensor, inputs|
786
- target_var, momentum_var, learning_rate, grad, momentum = inputs
787
- assign = tensor.inputs[0] || tensor
788
- assign_acc = tensor.inputs[1]
789
- assign_acc.value = process_vector_math_op(tensor, momentum_var, grad, context, ->(t, u) { t * momentum + u })
790
- if tensor.options[:use_nesterov]
791
- delta = process_vector_math_op(tensor, grad, momentum_var, context, ->(g, acc) { g * learning_rate + acc * momentum * learning_rate })
792
- assign.value = process_vector_math_op(tensor,target_var, delta, context, ->(t, u) { t - u })
793
- else
794
- assign.value = process_vector_math_op(tensor, target_var, momentum_var, context, ->(v, acc) { v - acc * learning_rate })
795
- end
796
- assign.value
797
- end
798
-
799
628
  register_op :broadcast_gradient_args do |_context, tensor, inputs|
800
629
  rx, ry = get_broadcast_gradient_args(inputs[0], inputs[1])
801
630
  OutputGroup.new([rx, ry], tensor.inputs.map(&:data_type))
@@ -812,7 +641,8 @@ module TensorStream
812
641
  end
813
642
 
814
643
  register_op :flow_group, noop: true do |context, tensor, inputs|
815
- inputs.collect { |input| global_eval(tensor, input, context) }
644
+ inputs.each { |input| global_eval(tensor, input, context) }
645
+ nil # no output
816
646
  end
817
647
 
818
648
  register_op :softmax do |_context, _tensor, inputs|
@@ -856,83 +686,6 @@ module TensorStream
856
686
  end
857
687
  end
858
688
 
859
- register_op :log_softmax do |_context, _tensor, inputs|
860
- input_shape = shape_eval(inputs[0])
861
- last_dimen_list = last_axis(inputs[0])
862
-
863
- func = lambda { |logits|
864
- c = logits.max
865
- transformed_logits = logits.map { |l| l - c }
866
- sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
867
- transformed_logits.map { |x| x - Math.log(sum) }
868
- }
869
-
870
- if input_shape.size == 1
871
- func.call(last_dimen_list)
872
- else
873
- arr = last_dimen_list.collect do |list|
874
- func.call(list)
875
- end
876
- TensorShape.reshape(arr.flatten, input_shape)
877
- end
878
- end
879
-
880
- register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context, tensor, inputs|
881
- last_dimen_list = last_axis(inputs[0])
882
- input_shape = shape_eval(inputs[0])
883
- rank = input_shape.size - 1
884
- labels = last_axis(inputs[1])
885
- func = lambda { |logits, label|
886
- c = logits.max
887
- transformed_logits = logits.map { |l| l - c }
888
- sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
889
- losses = transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
890
- probs = transformed_logits.zip(label).map { |x, y| (Math.exp(x) / sum) - y }
891
- [losses, probs]
892
- }
893
-
894
- if input_shape.size == 1
895
- loss, prob = func.call(last_dimen_list, labels)
896
- loss = reduce(loss, rank, false)
897
- OutputGroup.new([loss, prob], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
898
- else
899
- losses = []
900
- backprobs = []
901
- arr = last_dimen_list.zip(labels).each do |list, label|
902
- loss, prob = func.call(list, label)
903
- losses << loss
904
- backprobs << prob
905
- end
906
- reshaped_losses = TensorShape.reshape(losses.flatten, input_shape)
907
- reshaped_backprops = TensorShape.reshape(backprobs.flatten, input_shape)
908
- reshaped_losses = reduce(reshaped_losses, rank, false)
909
- OutputGroup.new([reshaped_losses, reshaped_backprops], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
910
- end
911
- end
912
-
913
- register_op :softmax_cross_entropy_with_logits_v2_grad do |_context, _tensor, inputs|
914
- last_dimen_list = last_axis(inputs[0])
915
- labels = last_axis(inputs[1])
916
- passed_grads = last_axis(inputs[2])
917
- input_shape = shape_eval(inputs[0])
918
-
919
- func = lambda { |logits, label, grad|
920
- c = logits.max
921
- transformed_logits = logits.map { |l| Math.exp(l - c) }
922
- e_sum = transformed_logits.reduce(:+)
923
- transformed_logits.zip(label).zip(grad).map { |(x, y), g| (x / e_sum) * g - y }
924
- }
925
-
926
- if input_shape.size == 1
927
- func.call(last_dimen_list, labels, passed_grads)
928
- else
929
- arr = last_dimen_list.zip(labels).zip(passed_grads).collect do |(list, label), passed_grad|
930
- func.call(list, label, passed_grad)
931
- end
932
- TensorShape.reshape(arr.flatten, input_shape)
933
- end
934
- end
935
-
936
689
  register_op :check_numerics do |context, tensor, inputs|
937
690
  message = tensor.options[:message]
938
691
  f = lambda { |t, _b|