tensor_stream 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +4 -0
- data/README.md +18 -5
- data/lib/tensor_stream/evaluator/opencl/kernels/apply_adam.cl +23 -0
- data/lib/tensor_stream/evaluator/opencl/opencl_evaluator.rb +51 -4
- data/lib/tensor_stream/evaluator/ruby/math_ops.rb +144 -0
- data/lib/tensor_stream/evaluator/ruby/nn_ops.rb +99 -0
- data/lib/tensor_stream/evaluator/ruby_evaluator.rb +6 -253
- data/lib/tensor_stream/ops.rb +2 -1
- data/lib/tensor_stream/session.rb +17 -8
- data/lib/tensor_stream/train/adam_optimizer.rb +87 -0
- data/lib/tensor_stream/train/gradient_descent_optimizer.rb +2 -1
- data/lib/tensor_stream/train/optimizer.rb +25 -2
- data/lib/tensor_stream/train/slot_creator.rb +1 -1
- data/lib/tensor_stream/trainer.rb +1 -0
- data/lib/tensor_stream/utils.rb +25 -4
- data/lib/tensor_stream/variable.rb +1 -1
- data/lib/tensor_stream/variable_scope.rb +7 -1
- data/lib/tensor_stream/version.rb +1 -1
- data/samples/iris.rb +9 -6
- data/samples/linear_regression.rb +6 -4
- data/samples/nearest_neighbor.rb +2 -2
- data/{test_samples → samples}/raw_neural_net_sample.rb +17 -20
- metadata +7 -8
- data/test_samples/error.graphml +0 -120
- data/test_samples/gradient_sample.graphml +0 -1255
- data/test_samples/neural_network_raw.py +0 -101
- data/test_samples/test.py +0 -46
- data/test_samples/test2.py +0 -87
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: a3c7d0a810a79ceedc0237379b105d7a9b598ba2513ef2d59ba3cec78d7b0da0
|
4
|
+
data.tar.gz: 7c0d90b27e548b72a86e88e7181f3d3b131a5fa3c6800000c743dd6e47d47b3b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a7a8a5607883d868da3ceaa9f870ac5c6d6809d45992a9ae0e4f26d5648bda2c4a26ea3abec506abd92362dcf3b63935b3f6acbcc70b605600307313f8c69f49
|
7
|
+
data.tar.gz: db8917bee53f91e1017b5fdb8b9ece8b50a1096d249cee40eed8c335f597f32e5ce81056230b8ce1b9acff4df728497fa13e896212cbdaccc023dbdb7ed3591e
|
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
|
|
4
4
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
5
5
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
6
6
|
|
7
|
+
|
8
|
+
## [0.8.1] - 2018-08-30
|
9
|
+
- [TRAINING] Added AdamOptimizer
|
10
|
+
|
7
11
|
## [0.8.0] - 2018-08-29
|
8
12
|
### Added
|
9
13
|
- [TRAINING] Added new supported optimizer, MomentumOptimizer loosely based on tensorflow's implementation (with nesterov support)
|
data/README.md
CHANGED
@@ -17,7 +17,20 @@ The goal of this gem is to have a high performance machine learning and compute
|
|
17
17
|
- eager execution (experimental)
|
18
18
|
- (08-08-2018) Load pbtext files from tensorflow (Graph.parse_from_string)
|
19
19
|
|
20
|
-
|
20
|
+
## Compatibility
|
21
|
+
|
22
|
+
TensorStream comes with a pure ruby and OpenCL implementation out of the box. The pure ruby implementation
|
23
|
+
is known to work with most ruby implementations including TruffleRuby, JRuby as well as jit enabled versions of mri (ruby-2.6.0).
|
24
|
+
|
25
|
+
OpenCL is supported only on mri implementations of ruby. This can be enabled by including the OpenCL evaluator (Make sure you have OpenCL drivers installed correctly on your system):
|
26
|
+
|
27
|
+
```ruby
|
28
|
+
require 'tensor_stream/evaluator/opencl/opencl_evaluator'
|
29
|
+
```
|
30
|
+
|
31
|
+
OpenCL is basically a requirement for deep learning and image processing tasks as the ruby implementation is too slow even with jit speedups using latest ruby implementations.
|
32
|
+
|
33
|
+
OpenCL kernels used by tensorstream can be found at tensor_stream/lib/evaluator/opencl/kernels. These are non specific and should work with any device that supports OpenCL including intel GPUs and CPUs, as well as GPUs from Nvidia and AMD.
|
21
34
|
|
22
35
|
## Installation
|
23
36
|
|
@@ -72,6 +85,8 @@ pred = X * W + b
|
|
72
85
|
# Mean squared error
|
73
86
|
cost = ((pred - Y) ** 2).reduce(:+) / ( 2 * n_samples)
|
74
87
|
|
88
|
+
# optimizer = TensorStream::Train::MomentumOptimizer.new(0.01, 0.5, use_nesterov: true).minimize(cost)
|
89
|
+
# optimizer = TensorStream::Train::AdamOptimizer.new.minimize(cost)
|
75
90
|
optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate).minimize(cost)
|
76
91
|
|
77
92
|
# Initialize the variables (i.e. assign their default value)
|
@@ -338,7 +353,7 @@ ruby 2.4
|
|
338
353
|
$ ruby -v
|
339
354
|
ruby 2.4.0p0 (2016-12-24 revision 57164) [x86_64-linux]
|
340
355
|
$ ruby samples/linear_regression.rb
|
341
|
-
495 seconds
|
356
|
+
495 seconds 10000 epochs
|
342
357
|
```
|
343
358
|
|
344
359
|
ruby 2.6.0-preview2
|
@@ -383,8 +398,6 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
383
398
|
|
384
399
|
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/tensor_stream. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
385
400
|
|
386
|
-
|
387
401
|
## License
|
388
402
|
|
389
|
-
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
390
|
-
|
403
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
@@ -0,0 +1,23 @@
|
|
1
|
+
% c_dtype = dtype_to_c_type(dtype)
|
2
|
+
// same dimension add floating point op
|
3
|
+
__kernel void apply_adam_<%= dtype %>(const int M, const int N,
|
4
|
+
__global const <%= c_dtype %> *grad,
|
5
|
+
__global const <%= c_dtype %> *learning_rate,
|
6
|
+
__global const <%= c_dtype %> *beta1_power,
|
7
|
+
__global const <%= c_dtype %> *beta2_power,
|
8
|
+
__global const <%= c_dtype %> *beta1,
|
9
|
+
__global const <%= c_dtype %> *beta2,
|
10
|
+
__global const <%= c_dtype %> *epsilon,
|
11
|
+
__global <%= c_dtype %> *momentum,
|
12
|
+
__global <%= c_dtype %> *output, __global <%= c_dtype %> *v) {
|
13
|
+
// Get the index of the current element to be processed
|
14
|
+
const int globalRow = get_global_id(0); // Row ID of C (0..M)
|
15
|
+
const int globalCol = get_global_id(1); // Col ID of C (0..N)
|
16
|
+
const int index = globalRow * N + globalCol;
|
17
|
+
|
18
|
+
<%= c_dtype %> alpha = learning_rate[0] * sqrt(1.0 - beta2_power[0]) / (1.0 - beta1_power[0]);
|
19
|
+
|
20
|
+
momentum[index] += (grad[index] - momentum[index]) * (1.0 - beta1[0]);
|
21
|
+
v[index] += (grad[index] * grad[index] - v[index]) * (1.0 - beta2[0]);
|
22
|
+
output[index] -= (momentum[index] * alpha) / ( sqrt(v[index]) + epsilon[0] );
|
23
|
+
}
|
@@ -107,9 +107,8 @@ module TensorStream
|
|
107
107
|
end
|
108
108
|
end
|
109
109
|
|
110
|
-
def
|
110
|
+
def enqueue_buffer_read(tensor, context)
|
111
111
|
buffer = _run(tensor, context)
|
112
|
-
|
113
112
|
if buffer.is_a?(Array)
|
114
113
|
buffer = buffer.collect do |b|
|
115
114
|
next b if b.buffer.size.zero?
|
@@ -122,7 +121,12 @@ module TensorStream
|
|
122
121
|
return [] if buffer.buffer.nil?
|
123
122
|
return buffer if buffer.buffer.size.zero?
|
124
123
|
_opencl_queue.enqueue_read_buffer(buffer.cl_buffer, buffer.buffer, event_wait_list: build_event_wait_list([buffer]))
|
124
|
+
buffer
|
125
125
|
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def complete_eval(tensor, context)
|
129
|
+
buffer = enqueue_buffer_read(tensor, context)
|
126
130
|
_opencl_queue.finish
|
127
131
|
buffer
|
128
132
|
end
|
@@ -339,6 +343,48 @@ module TensorStream
|
|
339
343
|
learning_rate.cl_buffer, momentum.cl_buffer, output_buffer.cl_buffer,
|
340
344
|
assign_acc.buffer.cl_buffer, event_wait_list: event_wait_list)
|
341
345
|
output_buffer.op = event
|
346
|
+
assign_acc.buffer.op = event
|
347
|
+
output_buffer
|
348
|
+
end
|
349
|
+
|
350
|
+
# Adam optimization algorithm
|
351
|
+
register_op :apply_adam do |_context, tensor, inputs|
|
352
|
+
_target_var, _m, _v, beta1_power, beta2_power, lr_t, beta1_t, beta2_t, epsilon_t, grad = inputs
|
353
|
+
|
354
|
+
assign = tensor.inputs[0] || tensor
|
355
|
+
assign_m = tensor.inputs[1]
|
356
|
+
assign_v = tensor.inputs[2]
|
357
|
+
|
358
|
+
# mark variable buffers as dirty
|
359
|
+
assign.buffer.dirty = true # force buffer copy when variable is read externally
|
360
|
+
assign_m.buffer.dirty = true # force buffer copy when variable is read externally
|
361
|
+
assign_v.buffer.dirty = true # force buffer copy when variable is read externally
|
362
|
+
|
363
|
+
output_buffer = assign.buffer
|
364
|
+
|
365
|
+
m, n = output_buffer.shape
|
366
|
+
work_group = [m || 1, n || 1]
|
367
|
+
cl_m = OpenCL::Int1.new(m || 1)
|
368
|
+
cl_n = OpenCL::Int1.new(n || 1)
|
369
|
+
|
370
|
+
event_wait_list = build_event_wait_list(inputs)
|
371
|
+
method_call = :"apply_adam_#{output_buffer.data_type}"
|
372
|
+
event = _cl_program("apply_adam", dtype: output_buffer.data_type)
|
373
|
+
.send(method_call, _opencl_queue, work_group, cl_m, cl_n,
|
374
|
+
grad.cl_buffer,
|
375
|
+
lr_t.cl_buffer,
|
376
|
+
beta1_power.cl_buffer,
|
377
|
+
beta2_power.cl_buffer,
|
378
|
+
beta1_t.cl_buffer,
|
379
|
+
beta2_t.cl_buffer,
|
380
|
+
epsilon_t.cl_buffer,
|
381
|
+
assign_m.buffer.cl_buffer,
|
382
|
+
assign.buffer.cl_buffer,
|
383
|
+
assign_v.buffer.cl_buffer,
|
384
|
+
event_wait_list: event_wait_list)
|
385
|
+
output_buffer.op = event
|
386
|
+
assign_m.buffer.op = event
|
387
|
+
assign_v.buffer.op = event
|
342
388
|
output_buffer
|
343
389
|
end
|
344
390
|
|
@@ -713,8 +759,9 @@ module TensorStream
|
|
713
759
|
convert_to_opencl(arr.buffer, shape, data_type: arr.data_type, name: tensor.name)
|
714
760
|
end
|
715
761
|
|
716
|
-
register_op :flow_group do |
|
717
|
-
|
762
|
+
register_op :flow_group do |context, _tensor, inputs|
|
763
|
+
_opencl_queue.finish
|
764
|
+
nil
|
718
765
|
end
|
719
766
|
|
720
767
|
register_op :size do |_context, tensor, inputs|
|
@@ -0,0 +1,144 @@
|
|
1
|
+
module TensorStream
|
2
|
+
module MathOps
|
3
|
+
def MathOps.included(klass)
|
4
|
+
klass.class_eval do
|
5
|
+
register_op :tanh, no_eval: true do |context, _tensor, inputs|
|
6
|
+
call_op(:tanh, inputs[0], context, ->(t, _b) { Math.tanh(t) })
|
7
|
+
end
|
8
|
+
|
9
|
+
register_op :tan, no_eval: true do |context, _tensor, inputs|
|
10
|
+
call_op(:tan, inputs[0], context, ->(t, _b) { Math.tan(t) })
|
11
|
+
end
|
12
|
+
|
13
|
+
register_op :atan, no_eval: true do |context, _tensor, inputs|
|
14
|
+
call_op(:atan, inputs[0], context, ->(t, _b) { Math.atan(t) })
|
15
|
+
end
|
16
|
+
|
17
|
+
register_op :sec, no_eval: true do |context, _tensor, inputs|
|
18
|
+
call_op(:sec, inputs[0], context, ->(t, _b) { Math.sec(t) })
|
19
|
+
end
|
20
|
+
|
21
|
+
register_op :sin, no_eval: true do |context, _tensor, inputs|
|
22
|
+
call_op(:sin, inputs[0], context, ->(t, _b) { Math.sin(t) })
|
23
|
+
end
|
24
|
+
|
25
|
+
register_op :add, no_eval: true do |context, tensor, inputs|
|
26
|
+
a, b = inputs
|
27
|
+
call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
|
28
|
+
end
|
29
|
+
|
30
|
+
register_op :add_n, no_eval: true do |context, tensor, inputs|
|
31
|
+
if inputs.size == 1
|
32
|
+
complete_eval(inputs[0], context)
|
33
|
+
elsif inputs.size > 1
|
34
|
+
|
35
|
+
a = inputs.pop
|
36
|
+
until inputs.empty?
|
37
|
+
b = inputs.pop
|
38
|
+
a = call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
|
39
|
+
end
|
40
|
+
a
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
register_op :sub, no_eval: true do |context, tensor, inputs|
|
45
|
+
a, b = inputs
|
46
|
+
call_vector_op(tensor, :sub, a, b, context, ->(t, u) { t - u })
|
47
|
+
end
|
48
|
+
|
49
|
+
register_op %i[floor_mod mod], no_eval: true do |context, tensor, inputs|
|
50
|
+
a, b = inputs
|
51
|
+
call_vector_op(tensor, :mod, a, b, context, ->(t, u) { t % u })
|
52
|
+
end
|
53
|
+
|
54
|
+
register_op %i[floor_div], no_eval: true do |context, tensor, inputs|
|
55
|
+
a, b = inputs
|
56
|
+
if fp_type?(tensor.data_type)
|
57
|
+
call_vector_op(tensor, :div, a, b, context, ->(t, u) { (t / u).to_i.to_f })
|
58
|
+
else
|
59
|
+
call_vector_op(tensor, :div, a, b, context, ->(t, u) { t / u })
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
register_op :mul, no_eval: true do |context, tensor, inputs|
|
64
|
+
a, b = inputs
|
65
|
+
call_vector_op(tensor, :mul, a, b, context, ->(t, u) { t * u })
|
66
|
+
end
|
67
|
+
|
68
|
+
register_op :pow, no_eval: true do |context, tensor, inputs|
|
69
|
+
a, b = inputs
|
70
|
+
call_vector_op(tensor, :pow, a, b, context, ->(t, u) { t**u })
|
71
|
+
end
|
72
|
+
|
73
|
+
register_op :squared_difference, no_eval: true do |context, tensor, inputs|
|
74
|
+
a, b = inputs
|
75
|
+
call_vector_op(tensor, :squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
|
76
|
+
end
|
77
|
+
|
78
|
+
register_op :round, no_eval: true do |context, _tensor, inputs|
|
79
|
+
call_op(:round, inputs[0], context, ->(t, _b) { t.round })
|
80
|
+
end
|
81
|
+
|
82
|
+
register_op :abs, no_eval: true do |context, _tensor, inputs|
|
83
|
+
call_op(:abs, inputs[0], context, ->(t, _b) { t.abs })
|
84
|
+
end
|
85
|
+
|
86
|
+
register_op :asin, no_eval: true do |context, _tensor, inputs|
|
87
|
+
call_op(:asin, inputs[0], context, ->(t, _b) { Math.asin(t) })
|
88
|
+
end
|
89
|
+
|
90
|
+
register_op :acos, no_eval: true do |context, _tensor, inputs|
|
91
|
+
call_op(:acos, inputs[0], context, ->(t, _b) { Math.acos(t) })
|
92
|
+
end
|
93
|
+
|
94
|
+
register_op :cos, no_eval: true do |context, _tensor, inputs|
|
95
|
+
call_op(:cos, inputs[0], context, ->(t, _b) { Math.cos(t) })
|
96
|
+
end
|
97
|
+
|
98
|
+
register_op :log1p, no_eval: true do |context, _tensor, inputs|
|
99
|
+
call_op(:log1p, inputs[0], context, ->(t, _b) { Math.log(1 + t) })
|
100
|
+
end
|
101
|
+
|
102
|
+
register_op :log, no_eval: true do |context, _tensor, inputs|
|
103
|
+
call_op(:log, inputs[0], context, ->(t, _b) { t < 0 ? Float::NAN : Math.log(t) })
|
104
|
+
end
|
105
|
+
|
106
|
+
register_op :exp, no_eval: true do |context, _tensor, inputs|
|
107
|
+
call_op(:exp, inputs[0], context, ->(t, _b) { Math.exp(t) })
|
108
|
+
end
|
109
|
+
|
110
|
+
register_op :sigmoid, no_eval: true do |context, _tensor, inputs|
|
111
|
+
call_op(:sigmoid, inputs[0], context, ->(t, _b) { sigmoid(t) })
|
112
|
+
end
|
113
|
+
|
114
|
+
register_op :sqrt, no_eval: true do |context, _tensor, inputs|
|
115
|
+
call_op(:sqrt, inputs[0], context, ->(t, _b) { Math.sqrt(t) })
|
116
|
+
end
|
117
|
+
|
118
|
+
register_op :floor, no_eval: true do |context, _tensor, inputs|
|
119
|
+
call_op(:floor, inputs[0], context, ->(t, _b) { t.floor })
|
120
|
+
end
|
121
|
+
|
122
|
+
register_op :ceil, no_eval: true do |context, _tensor, inputs|
|
123
|
+
call_op(:ceil, inputs[0], context, ->(t, _b) { t.ceil })
|
124
|
+
end
|
125
|
+
|
126
|
+
register_op :square, no_eval: true do |context, _tensor, inputs|
|
127
|
+
call_op(:square, inputs[0], context, ->(t, _b) { t * t })
|
128
|
+
end
|
129
|
+
|
130
|
+
register_op :reciprocal, no_eval: true do |context, _tensor, inputs|
|
131
|
+
call_op(:reciprocal, inputs[0], context, ->(t, _b) { 1 / t })
|
132
|
+
end
|
133
|
+
|
134
|
+
register_op %i[neg negate], no_eval: true do |context, tensor, inputs|
|
135
|
+
call_vector_op(tensor, :negate, inputs[0], nil, context, ->(t, _u) { -t })
|
136
|
+
end
|
137
|
+
|
138
|
+
register_op :tanh_grad, no_eval: true do |context, _tensor, inputs|
|
139
|
+
call_op(:tanh_grad, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
module TensorStream
|
2
|
+
## Collection of machine learning related ops
|
3
|
+
module NNOps
|
4
|
+
def NNOps.included(klass)
|
5
|
+
klass.class_eval do
|
6
|
+
register_op :apply_gradient_descent do |context, tensor, inputs|
|
7
|
+
target_var, learning_rate, delta = inputs
|
8
|
+
assign = tensor.inputs[0] || tensor
|
9
|
+
|
10
|
+
assign.value = process_vector_math_op(tensor, target_var, delta, context, ->(t, u) { t - u * learning_rate })
|
11
|
+
assign.value
|
12
|
+
end
|
13
|
+
|
14
|
+
register_op :apply_momentum do |context, tensor, inputs|
|
15
|
+
target_var, momentum_var, learning_rate, grad, momentum = inputs
|
16
|
+
assign = tensor.inputs[0] || tensor
|
17
|
+
assign_acc = tensor.inputs[1]
|
18
|
+
assign_acc.value = process_vector_math_op(tensor, momentum_var, grad, context, ->(t, u) { t * momentum + u })
|
19
|
+
if tensor.options[:use_nesterov]
|
20
|
+
delta = process_vector_math_op(tensor, grad, momentum_var, context, ->(g, acc) { g * learning_rate + acc * momentum * learning_rate })
|
21
|
+
assign.value = process_vector_math_op(tensor,target_var, delta, context, ->(t, u) { t - u })
|
22
|
+
else
|
23
|
+
assign.value = process_vector_math_op(tensor, target_var, momentum_var, context, ->(v, acc) { v - acc * learning_rate })
|
24
|
+
end
|
25
|
+
assign.value
|
26
|
+
end
|
27
|
+
|
28
|
+
register_op :apply_adam do |context, tensor, inputs|
|
29
|
+
target_var, m, v, beta1_power, beta2_power, lr_t, beta1_t, beta2_t, epsilon_t, grad = inputs
|
30
|
+
alpha = lr_t * Math.sqrt( 1.0 - beta2_power) / (1.0 - beta1_power)
|
31
|
+
assign = tensor.inputs[0]
|
32
|
+
assign_m = tensor.inputs[1]
|
33
|
+
assign_v = tensor.inputs[2]
|
34
|
+
|
35
|
+
m_delta = process_vector_math_op(tensor, grad, m, context, ->(g, m_d) { (g - m_d) * (1.0 - beta1_t) })
|
36
|
+
assign_m.value = process_vector_math_op(tensor, m, m_delta, context, ->(u_d , v_d) { u_d + v_d })
|
37
|
+
assign_v.value = process_vector_math_op(tensor, v, grad, context, ->(u_d , v_d) { u_d + (v_d ** 2 - u_d) * (1.0 - beta2_t)})
|
38
|
+
v_delta = process_vector_math_op(tensor, assign_m.value, assign_v.value, context, ->(m_d , v_d) { (m_d * alpha) / (Math.sqrt(v_d) + epsilon_t) })
|
39
|
+
assign.value = process_vector_math_op(tensor, target_var, v_delta, context, ->(var_d , delta_d) { var_d - delta_d })
|
40
|
+
assign.value
|
41
|
+
end
|
42
|
+
|
43
|
+
register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context, tensor, inputs|
|
44
|
+
last_dimen_list = last_axis(inputs[0])
|
45
|
+
input_shape = shape_eval(inputs[0])
|
46
|
+
rank = input_shape.size - 1
|
47
|
+
labels = last_axis(inputs[1])
|
48
|
+
func = lambda { |logits, label|
|
49
|
+
c = logits.max
|
50
|
+
transformed_logits = logits.map { |l| l - c }
|
51
|
+
sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
|
52
|
+
losses = transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
|
53
|
+
probs = transformed_logits.zip(label).map { |x, y| (Math.exp(x) / sum) - y }
|
54
|
+
[losses, probs]
|
55
|
+
}
|
56
|
+
|
57
|
+
if input_shape.size == 1
|
58
|
+
loss, prob = func.call(last_dimen_list, labels)
|
59
|
+
loss = reduce(loss, rank, false)
|
60
|
+
TensorStream::Evaluator::OutputGroup.new([loss, prob], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
|
61
|
+
else
|
62
|
+
losses = []
|
63
|
+
backprobs = []
|
64
|
+
arr = last_dimen_list.zip(labels).each do |list, label|
|
65
|
+
loss, prob = func.call(list, label)
|
66
|
+
losses << loss
|
67
|
+
backprobs << prob
|
68
|
+
end
|
69
|
+
reshaped_losses = TensorShape.reshape(losses.flatten, input_shape)
|
70
|
+
reshaped_backprops = TensorShape.reshape(backprobs.flatten, input_shape)
|
71
|
+
reshaped_losses = reduce(reshaped_losses, rank, false)
|
72
|
+
TensorStream::Evaluator::OutputGroup.new([reshaped_losses, reshaped_backprops], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
register_op :log_softmax do |_context, _tensor, inputs|
|
77
|
+
input_shape = shape_eval(inputs[0])
|
78
|
+
last_dimen_list = last_axis(inputs[0])
|
79
|
+
|
80
|
+
func = lambda { |logits|
|
81
|
+
c = logits.max
|
82
|
+
transformed_logits = logits.map { |l| l - c }
|
83
|
+
sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
|
84
|
+
transformed_logits.map { |x| x - Math.log(sum) }
|
85
|
+
}
|
86
|
+
|
87
|
+
if input_shape.size == 1
|
88
|
+
func.call(last_dimen_list)
|
89
|
+
else
|
90
|
+
arr = last_dimen_list.collect do |list|
|
91
|
+
func.call(list)
|
92
|
+
end
|
93
|
+
TensorShape.reshape(arr.flatten, input_shape)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -2,6 +2,8 @@ require 'tensor_stream/evaluator/operation_helpers/random_gaussian'
|
|
2
2
|
require 'tensor_stream/evaluator/operation_helpers/array_ops_helper'
|
3
3
|
require 'tensor_stream/evaluator/operation_helpers/math_helper'
|
4
4
|
require 'tensor_stream/evaluator/base_evaluator'
|
5
|
+
require 'tensor_stream/evaluator/ruby/math_ops'
|
6
|
+
require 'tensor_stream/evaluator/ruby/nn_ops'
|
5
7
|
|
6
8
|
module TensorStream
|
7
9
|
module Evaluator
|
@@ -29,6 +31,8 @@ module TensorStream
|
|
29
31
|
include TensorStream::OpHelper
|
30
32
|
include TensorStream::ArrayOpsHelper
|
31
33
|
include TensorStream::MathHelper
|
34
|
+
include TensorStream::MathOps
|
35
|
+
include TensorStream::NNOps
|
32
36
|
|
33
37
|
def run(tensor, execution_context)
|
34
38
|
return tensor.map { |t| run(t, execution_context) } if tensor.is_a?(Array) && !tensor.empty? && tensor[0].is_a?(Tensor)
|
@@ -243,143 +247,10 @@ module TensorStream
|
|
243
247
|
Tensor.cast_dtype(input.flatten.size, tensor.options[:out_type])
|
244
248
|
end
|
245
249
|
|
246
|
-
register_op %i[neg negate], no_eval: true do |context, tensor, inputs|
|
247
|
-
call_vector_op(tensor, :negate, inputs[0], nil, context, ->(t, _u) { -t })
|
248
|
-
end
|
249
|
-
|
250
|
-
register_op :add, no_eval: true do |context, tensor, inputs|
|
251
|
-
a, b = inputs
|
252
|
-
call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
|
253
|
-
end
|
254
|
-
|
255
|
-
register_op :add_n, no_eval: true do |context, tensor, inputs|
|
256
|
-
if inputs.size == 1
|
257
|
-
complete_eval(inputs[0], context)
|
258
|
-
elsif inputs.size > 1
|
259
|
-
|
260
|
-
a = inputs.pop
|
261
|
-
until inputs.empty?
|
262
|
-
b = inputs.pop
|
263
|
-
a = call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
|
264
|
-
end
|
265
|
-
a
|
266
|
-
end
|
267
|
-
end
|
268
|
-
|
269
|
-
register_op :sub, no_eval: true do |context, tensor, inputs|
|
270
|
-
a, b = inputs
|
271
|
-
call_vector_op(tensor, :sub, a, b, context, ->(t, u) { t - u })
|
272
|
-
end
|
273
|
-
|
274
|
-
register_op %i[floor_mod mod], no_eval: true do |context, tensor, inputs|
|
275
|
-
a, b = inputs
|
276
|
-
call_vector_op(tensor, :mod, a, b, context, ->(t, u) { t % u })
|
277
|
-
end
|
278
|
-
|
279
|
-
register_op %i[floor_div], no_eval: true do |context, tensor, inputs|
|
280
|
-
a, b = inputs
|
281
|
-
if fp_type?(tensor.data_type)
|
282
|
-
call_vector_op(tensor, :div, a, b, context, ->(t, u) { (t / u).to_i.to_f })
|
283
|
-
else
|
284
|
-
call_vector_op(tensor, :div, a, b, context, ->(t, u) { t / u })
|
285
|
-
end
|
286
|
-
end
|
287
|
-
|
288
|
-
register_op :mul, no_eval: true do |context, tensor, inputs|
|
289
|
-
a, b = inputs
|
290
|
-
call_vector_op(tensor, :mul, a, b, context, ->(t, u) { t * u })
|
291
|
-
end
|
292
|
-
|
293
|
-
register_op :pow, no_eval: true do |context, tensor, inputs|
|
294
|
-
a, b = inputs
|
295
|
-
call_vector_op(tensor, :pow, a, b, context, ->(t, u) { t**u })
|
296
|
-
end
|
297
|
-
|
298
|
-
register_op :squared_difference, no_eval: true do |context, tensor, inputs|
|
299
|
-
a, b = inputs
|
300
|
-
call_vector_op(tensor, :squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
|
301
|
-
end
|
302
|
-
|
303
250
|
register_op %i[concat concat_v2] do |_context, tensor, inputs|
|
304
251
|
concat_array(inputs, tensor.options[:axis])
|
305
252
|
end
|
306
253
|
|
307
|
-
register_op :round, no_eval: true do |context, _tensor, inputs|
|
308
|
-
call_op(:round, inputs[0], context, ->(t, _b) { t.round })
|
309
|
-
end
|
310
|
-
|
311
|
-
register_op :abs, no_eval: true do |context, _tensor, inputs|
|
312
|
-
call_op(:abs, inputs[0], context, ->(t, _b) { t.abs })
|
313
|
-
end
|
314
|
-
|
315
|
-
register_op :tanh, no_eval: true do |context, _tensor, inputs|
|
316
|
-
call_op(:tanh, inputs[0], context, ->(t, _b) { Math.tanh(t) })
|
317
|
-
end
|
318
|
-
|
319
|
-
register_op :tan, no_eval: true do |context, _tensor, inputs|
|
320
|
-
call_op(:tan, inputs[0], context, ->(t, _b) { Math.tan(t) })
|
321
|
-
end
|
322
|
-
|
323
|
-
register_op :atan, no_eval: true do |context, _tensor, inputs|
|
324
|
-
call_op(:atan, inputs[0], context, ->(t, _b) { Math.atan(t) })
|
325
|
-
end
|
326
|
-
|
327
|
-
register_op :sec, no_eval: true do |context, _tensor, inputs|
|
328
|
-
call_op(:sec, inputs[0], context, ->(t, _b) { Math.sec(t) })
|
329
|
-
end
|
330
|
-
|
331
|
-
register_op :sin, no_eval: true do |context, _tensor, inputs|
|
332
|
-
call_op(:sin, inputs[0], context, ->(t, _b) { Math.sin(t) })
|
333
|
-
end
|
334
|
-
|
335
|
-
register_op :asin, no_eval: true do |context, _tensor, inputs|
|
336
|
-
call_op(:asin, inputs[0], context, ->(t, _b) { Math.asin(t) })
|
337
|
-
end
|
338
|
-
|
339
|
-
register_op :acos, no_eval: true do |context, _tensor, inputs|
|
340
|
-
call_op(:acos, inputs[0], context, ->(t, _b) { Math.acos(t) })
|
341
|
-
end
|
342
|
-
|
343
|
-
register_op :cos, no_eval: true do |context, _tensor, inputs|
|
344
|
-
call_op(:cos, inputs[0], context, ->(t, _b) { Math.cos(t) })
|
345
|
-
end
|
346
|
-
|
347
|
-
register_op :log1p, no_eval: true do |context, _tensor, inputs|
|
348
|
-
call_op(:log1p, inputs[0], context, ->(t, _b) { Math.log(1 + t) })
|
349
|
-
end
|
350
|
-
|
351
|
-
register_op :log, no_eval: true do |context, _tensor, inputs|
|
352
|
-
call_op(:log, inputs[0], context, ->(t, _b) { t < 0 ? Float::NAN : Math.log(t) })
|
353
|
-
end
|
354
|
-
|
355
|
-
register_op :exp, no_eval: true do |context, _tensor, inputs|
|
356
|
-
call_op(:exp, inputs[0], context, ->(t, _b) { Math.exp(t) })
|
357
|
-
end
|
358
|
-
|
359
|
-
register_op :sigmoid, no_eval: true do |context, _tensor, inputs|
|
360
|
-
call_op(:sigmoid, inputs[0], context, ->(t, _b) { sigmoid(t) })
|
361
|
-
end
|
362
|
-
|
363
|
-
register_op :sqrt, no_eval: true do |context, _tensor, inputs|
|
364
|
-
call_op(:sqrt, inputs[0], context, ->(t, _b) { Math.sqrt(t) })
|
365
|
-
end
|
366
|
-
|
367
|
-
register_op :floor, no_eval: true do |context, _tensor, inputs|
|
368
|
-
call_op(:floor, inputs[0], context, ->(t, _b) { t.floor })
|
369
|
-
end
|
370
|
-
|
371
|
-
register_op :ceil, no_eval: true do |context, _tensor, inputs|
|
372
|
-
call_op(:ceil, inputs[0], context, ->(t, _b) { t.ceil })
|
373
|
-
end
|
374
|
-
|
375
|
-
register_op :square, no_eval: true do |context, _tensor, inputs|
|
376
|
-
call_op(:square, inputs[0], context, ->(t, _b) { t * t })
|
377
|
-
end
|
378
|
-
|
379
|
-
register_op :reciprocal, no_eval: true do |context, _tensor, inputs|
|
380
|
-
call_op(:reciprocal, inputs[0], context, ->(t, _b) { 1 / t })
|
381
|
-
end
|
382
|
-
|
383
254
|
register_op :stop_gradient, no_eval: true do |_context, _tensor, inputs|
|
384
255
|
inputs[0]
|
385
256
|
end
|
@@ -517,14 +388,6 @@ module TensorStream
|
|
517
388
|
end
|
518
389
|
|
519
390
|
register_op :sum, noop: true do |context, tensor, _inputs|
|
520
|
-
# axis = complete_eval(tensor.inputs[1], context)
|
521
|
-
# # fast path
|
522
|
-
# if axis.nil? && !tensor.options[:keepdims]
|
523
|
-
# arr = complete_eval(tensor.inputs[0], context)
|
524
|
-
# next arr unless arr.is_a?(Array)
|
525
|
-
# next arr.flatten.reduce(:+)
|
526
|
-
# end
|
527
|
-
|
528
391
|
func = lambda do |arr|
|
529
392
|
reduced_val = arr[0]
|
530
393
|
arr[1..arr.size].each do |v|
|
@@ -537,14 +400,6 @@ module TensorStream
|
|
537
400
|
end
|
538
401
|
|
539
402
|
register_op :prod, noop: true do |context, tensor, _inputs|
|
540
|
-
# axis = complete_eval(tensor.inputs[1], context)
|
541
|
-
# # fast path
|
542
|
-
# if axis.nil? && !tensor.options[:keepdims]
|
543
|
-
# arr = complete_eval(tensor.inputs[0], context)
|
544
|
-
# next arr unless arr.is_a?(Array)
|
545
|
-
# next arr.flatten.reduce(:*)
|
546
|
-
# end
|
547
|
-
|
548
403
|
c = fp_type?(tensor.data_type) ? 1.0 : 1
|
549
404
|
func = lambda do |arr|
|
550
405
|
return c if arr.nil?
|
@@ -577,10 +432,6 @@ module TensorStream
|
|
577
432
|
r
|
578
433
|
end
|
579
434
|
|
580
|
-
register_op :tanh_grad, no_eval: true do |context, _tensor, inputs|
|
581
|
-
call_op(:tanh_grad, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
|
582
|
-
end
|
583
|
-
|
584
435
|
register_op :transpose do |_context, tensor, inputs|
|
585
436
|
shape = shape_eval(inputs[0])
|
586
437
|
rank = get_rank(inputs[0])
|
@@ -774,28 +625,6 @@ module TensorStream
|
|
774
625
|
call_vector_op(tensor, :min, inputs[0], inputs[1], context, ->(t, u) { [t, u].min })
|
775
626
|
end
|
776
627
|
|
777
|
-
register_op :apply_gradient_descent do |context, tensor, inputs|
|
778
|
-
target_var, learning_rate, delta = inputs
|
779
|
-
assign = tensor.inputs[0] || tensor
|
780
|
-
|
781
|
-
assign.value = process_vector_math_op(tensor, target_var, delta, context, ->(t, u) { t - u * learning_rate })
|
782
|
-
assign.value
|
783
|
-
end
|
784
|
-
|
785
|
-
register_op :apply_momentum do |context, tensor, inputs|
|
786
|
-
target_var, momentum_var, learning_rate, grad, momentum = inputs
|
787
|
-
assign = tensor.inputs[0] || tensor
|
788
|
-
assign_acc = tensor.inputs[1]
|
789
|
-
assign_acc.value = process_vector_math_op(tensor, momentum_var, grad, context, ->(t, u) { t * momentum + u })
|
790
|
-
if tensor.options[:use_nesterov]
|
791
|
-
delta = process_vector_math_op(tensor, grad, momentum_var, context, ->(g, acc) { g * learning_rate + acc * momentum * learning_rate })
|
792
|
-
assign.value = process_vector_math_op(tensor,target_var, delta, context, ->(t, u) { t - u })
|
793
|
-
else
|
794
|
-
assign.value = process_vector_math_op(tensor, target_var, momentum_var, context, ->(v, acc) { v - acc * learning_rate })
|
795
|
-
end
|
796
|
-
assign.value
|
797
|
-
end
|
798
|
-
|
799
628
|
register_op :broadcast_gradient_args do |_context, tensor, inputs|
|
800
629
|
rx, ry = get_broadcast_gradient_args(inputs[0], inputs[1])
|
801
630
|
OutputGroup.new([rx, ry], tensor.inputs.map(&:data_type))
|
@@ -812,7 +641,8 @@ module TensorStream
|
|
812
641
|
end
|
813
642
|
|
814
643
|
register_op :flow_group, noop: true do |context, tensor, inputs|
|
815
|
-
inputs.
|
644
|
+
inputs.each { |input| global_eval(tensor, input, context) }
|
645
|
+
nil # no output
|
816
646
|
end
|
817
647
|
|
818
648
|
register_op :softmax do |_context, _tensor, inputs|
|
@@ -856,83 +686,6 @@ module TensorStream
|
|
856
686
|
end
|
857
687
|
end
|
858
688
|
|
859
|
-
register_op :log_softmax do |_context, _tensor, inputs|
|
860
|
-
input_shape = shape_eval(inputs[0])
|
861
|
-
last_dimen_list = last_axis(inputs[0])
|
862
|
-
|
863
|
-
func = lambda { |logits|
|
864
|
-
c = logits.max
|
865
|
-
transformed_logits = logits.map { |l| l - c }
|
866
|
-
sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
|
867
|
-
transformed_logits.map { |x| x - Math.log(sum) }
|
868
|
-
}
|
869
|
-
|
870
|
-
if input_shape.size == 1
|
871
|
-
func.call(last_dimen_list)
|
872
|
-
else
|
873
|
-
arr = last_dimen_list.collect do |list|
|
874
|
-
func.call(list)
|
875
|
-
end
|
876
|
-
TensorShape.reshape(arr.flatten, input_shape)
|
877
|
-
end
|
878
|
-
end
|
879
|
-
|
880
|
-
register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context, tensor, inputs|
|
881
|
-
last_dimen_list = last_axis(inputs[0])
|
882
|
-
input_shape = shape_eval(inputs[0])
|
883
|
-
rank = input_shape.size - 1
|
884
|
-
labels = last_axis(inputs[1])
|
885
|
-
func = lambda { |logits, label|
|
886
|
-
c = logits.max
|
887
|
-
transformed_logits = logits.map { |l| l - c }
|
888
|
-
sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
|
889
|
-
losses = transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
|
890
|
-
probs = transformed_logits.zip(label).map { |x, y| (Math.exp(x) / sum) - y }
|
891
|
-
[losses, probs]
|
892
|
-
}
|
893
|
-
|
894
|
-
if input_shape.size == 1
|
895
|
-
loss, prob = func.call(last_dimen_list, labels)
|
896
|
-
loss = reduce(loss, rank, false)
|
897
|
-
OutputGroup.new([loss, prob], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
|
898
|
-
else
|
899
|
-
losses = []
|
900
|
-
backprobs = []
|
901
|
-
arr = last_dimen_list.zip(labels).each do |list, label|
|
902
|
-
loss, prob = func.call(list, label)
|
903
|
-
losses << loss
|
904
|
-
backprobs << prob
|
905
|
-
end
|
906
|
-
reshaped_losses = TensorShape.reshape(losses.flatten, input_shape)
|
907
|
-
reshaped_backprops = TensorShape.reshape(backprobs.flatten, input_shape)
|
908
|
-
reshaped_losses = reduce(reshaped_losses, rank, false)
|
909
|
-
OutputGroup.new([reshaped_losses, reshaped_backprops], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
|
910
|
-
end
|
911
|
-
end
|
912
|
-
|
913
|
-
register_op :softmax_cross_entropy_with_logits_v2_grad do |_context, _tensor, inputs|
|
914
|
-
last_dimen_list = last_axis(inputs[0])
|
915
|
-
labels = last_axis(inputs[1])
|
916
|
-
passed_grads = last_axis(inputs[2])
|
917
|
-
input_shape = shape_eval(inputs[0])
|
918
|
-
|
919
|
-
func = lambda { |logits, label, grad|
|
920
|
-
c = logits.max
|
921
|
-
transformed_logits = logits.map { |l| Math.exp(l - c) }
|
922
|
-
e_sum = transformed_logits.reduce(:+)
|
923
|
-
transformed_logits.zip(label).zip(grad).map { |(x, y), g| (x / e_sum) * g - y }
|
924
|
-
}
|
925
|
-
|
926
|
-
if input_shape.size == 1
|
927
|
-
func.call(last_dimen_list, labels, passed_grads)
|
928
|
-
else
|
929
|
-
arr = last_dimen_list.zip(labels).zip(passed_grads).collect do |(list, label), passed_grad|
|
930
|
-
func.call(list, label, passed_grad)
|
931
|
-
end
|
932
|
-
TensorShape.reshape(arr.flatten, input_shape)
|
933
|
-
end
|
934
|
-
end
|
935
|
-
|
936
689
|
register_op :check_numerics do |context, tensor, inputs|
|
937
690
|
message = tensor.options[:message]
|
938
691
|
f = lambda { |t, _b|
|