tensor_stream 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.rubocop.yml +6 -1
- data/CHANGELOG.md +10 -0
- data/README.md +35 -0
- data/lib/tensor_stream.rb +2 -2
- data/lib/tensor_stream/debugging/debugging.rb +2 -1
- data/lib/tensor_stream/dynamic_stitch.rb +23 -24
- data/lib/tensor_stream/evaluator/base_evaluator.rb +27 -18
- data/lib/tensor_stream/evaluator/opencl/kernels/apply_momentum.cl +16 -0
- data/lib/tensor_stream/evaluator/opencl/kernels/pack.cl +24 -0
- data/lib/tensor_stream/evaluator/opencl/kernels/softmax_cross.cl +6 -1
- data/lib/tensor_stream/evaluator/opencl/opencl_buffer.rb +6 -6
- data/lib/tensor_stream/evaluator/opencl/opencl_evaluator.rb +237 -107
- data/lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb +97 -7
- data/lib/tensor_stream/evaluator/ruby_evaluator.rb +230 -123
- data/lib/tensor_stream/exceptions.rb +1 -0
- data/lib/tensor_stream/graph_builder.rb +2 -3
- data/lib/tensor_stream/graph_deserializers/protobuf.rb +22 -23
- data/lib/tensor_stream/graph_serializers/graphml.rb +26 -29
- data/lib/tensor_stream/graph_serializers/pbtext.rb +22 -19
- data/lib/tensor_stream/helpers/string_helper.rb +4 -5
- data/lib/tensor_stream/math_gradients.rb +141 -77
- data/lib/tensor_stream/nn/nn_ops.rb +4 -6
- data/lib/tensor_stream/operation.rb +139 -120
- data/lib/tensor_stream/ops.rb +36 -3
- data/lib/tensor_stream/session.rb +7 -11
- data/lib/tensor_stream/tensor.rb +3 -3
- data/lib/tensor_stream/tensor_shape.rb +5 -0
- data/lib/tensor_stream/train/gradient_descent_optimizer.rb +4 -37
- data/lib/tensor_stream/train/momentum_optimizer.rb +48 -0
- data/lib/tensor_stream/train/optimizer.rb +129 -0
- data/lib/tensor_stream/train/saver.rb +0 -1
- data/lib/tensor_stream/train/slot_creator.rb +62 -0
- data/lib/tensor_stream/train/utils.rb +11 -12
- data/lib/tensor_stream/trainer.rb +3 -0
- data/lib/tensor_stream/utils.rb +18 -11
- data/lib/tensor_stream/variable.rb +19 -12
- data/lib/tensor_stream/variable_scope.rb +1 -1
- data/lib/tensor_stream/version.rb +1 -1
- data/samples/iris.rb +2 -1
- data/samples/linear_regression.rb +3 -1
- data/samples/nearest_neighbor.rb +2 -0
- data/test_samples/neural_network_raw.py +101 -0
- data/test_samples/raw_neural_net_sample.rb +6 -4
- data/test_samples/test2.py +73 -27
- metadata +9 -3
@@ -6,11 +6,11 @@ module TensorStream
|
|
6
6
|
start_index = start.shift
|
7
7
|
dimen_size = start_index + size.shift
|
8
8
|
|
9
|
-
input[start_index...dimen_size].collect do |
|
10
|
-
if
|
11
|
-
slice_tensor(
|
9
|
+
input[start_index...dimen_size].collect do |item|
|
10
|
+
if item.is_a?(Array)
|
11
|
+
slice_tensor(item, start.dup, size.dup)
|
12
12
|
else
|
13
|
-
|
13
|
+
item
|
14
14
|
end
|
15
15
|
end
|
16
16
|
end
|
@@ -25,7 +25,7 @@ module TensorStream
|
|
25
25
|
|
26
26
|
def _reduced_shape(input_shape, axes)
|
27
27
|
return [] if axes.nil? # reduce to scalar
|
28
|
-
axes = [
|
28
|
+
axes = [axes] unless axes.is_a?(Array)
|
29
29
|
return input_shape if axes.empty?
|
30
30
|
|
31
31
|
axes.each do |dimen|
|
@@ -72,8 +72,8 @@ module TensorStream
|
|
72
72
|
d = dims.shift
|
73
73
|
|
74
74
|
if input.is_a?(Array) && (get_rank(input) - 1) == dims.size
|
75
|
-
row_to_dup = input.collect do |
|
76
|
-
broadcast_dimensions(
|
75
|
+
row_to_dup = input.collect do |item|
|
76
|
+
broadcast_dimensions(item, dims.dup)
|
77
77
|
end
|
78
78
|
|
79
79
|
row_to_dup + Array.new(d) { row_to_dup }.flatten(1)
|
@@ -205,5 +205,95 @@ module TensorStream
|
|
205
205
|
end
|
206
206
|
end
|
207
207
|
end
|
208
|
+
|
209
|
+
def gather(params, indexes)
|
210
|
+
indexes.collect do |index|
|
211
|
+
if index.is_a?(Array)
|
212
|
+
gather(params, index)
|
213
|
+
else
|
214
|
+
params[index]
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
# general case transposition with flat arrays
|
220
|
+
def transpose_with_perm(arr, new_arr, shape, new_shape, perm)
|
221
|
+
arr_size = shape.reduce(:*)
|
222
|
+
divisors = shape.dup.drop(1).reverse.inject([1]) do |a, s|
|
223
|
+
a << s * a.last
|
224
|
+
end.reverse
|
225
|
+
|
226
|
+
multipliers = new_shape.dup.drop(1).reverse.inject([1]) do |a, s|
|
227
|
+
a << s * a.last
|
228
|
+
end.reverse
|
229
|
+
|
230
|
+
arr_size.times do |p|
|
231
|
+
ptr = p
|
232
|
+
index = []
|
233
|
+
divisors.each_with_object(index) do |div, a|
|
234
|
+
a << (ptr / div.to_f).floor
|
235
|
+
ptr = ptr % div
|
236
|
+
end
|
237
|
+
|
238
|
+
# remap based on perm
|
239
|
+
remaped = perm.map { |x| index[x] }
|
240
|
+
|
241
|
+
ptr2 = 0
|
242
|
+
multipliers.each_with_index do |m, idx|
|
243
|
+
ptr2 += remaped[idx] * m
|
244
|
+
end
|
245
|
+
|
246
|
+
new_arr[ptr2] = arr[p]
|
247
|
+
end
|
248
|
+
|
249
|
+
[new_arr, new_shape]
|
250
|
+
end
|
251
|
+
|
252
|
+
def reduce_axis(current_axis, axis, val, keep_dims, f)
|
253
|
+
return val unless val.is_a?(Array)
|
254
|
+
|
255
|
+
r = val.collect do |v|
|
256
|
+
reduce_axis(current_axis + 1, axis, v, keep_dims, f)
|
257
|
+
end
|
258
|
+
|
259
|
+
should_reduce_axis = axis.nil? || (axis.is_a?(Array) && axis.include?(current_axis)) || (current_axis == axis)
|
260
|
+
|
261
|
+
if should_reduce_axis
|
262
|
+
reduced_val = r[0]
|
263
|
+
if r.size > 1
|
264
|
+
reduced_val = f.call(r[0..val.size])
|
265
|
+
elsif r.empty?
|
266
|
+
reduced_val = f.call(nil)
|
267
|
+
end
|
268
|
+
keep_dims ? [reduced_val] : reduced_val
|
269
|
+
else
|
270
|
+
r
|
271
|
+
end
|
272
|
+
end
|
273
|
+
|
274
|
+
def reduce(val, axis, keep_dims, func = nil)
|
275
|
+
rank = get_rank(val)
|
276
|
+
return val if axis && axis.is_a?(Array) && axis.empty?
|
277
|
+
|
278
|
+
func = lambda do |arr|
|
279
|
+
reduced_val = arr[0]
|
280
|
+
arr[1..arr.size].each do |v|
|
281
|
+
reduced_val = vector_op(reduced_val, v, ->(t, u) { t + u })
|
282
|
+
end
|
283
|
+
reduced_val
|
284
|
+
end if func.nil?
|
285
|
+
|
286
|
+
axis = if axis.nil?
|
287
|
+
nil
|
288
|
+
elsif axis.is_a?(Array)
|
289
|
+
return val if axis.empty?
|
290
|
+
|
291
|
+
axis.map { |a| a < 0 ? rank - a.abs : a }
|
292
|
+
else
|
293
|
+
axis < 0 ? rank - axis.abs : axis
|
294
|
+
end
|
295
|
+
|
296
|
+
reduce_axis(0, axis, val, keep_dims, func)
|
297
|
+
end
|
208
298
|
end
|
209
299
|
end
|
@@ -139,16 +139,16 @@ module TensorStream
|
|
139
139
|
call_op(:sign, inputs[0], context, func)
|
140
140
|
end
|
141
141
|
|
142
|
-
register_op(:logical_and) do |context,
|
143
|
-
call_vector_op(:logical_and, inputs[0], inputs[1], context, ->(t, u) { t && u })
|
142
|
+
register_op(:logical_and) do |context, tensor, inputs|
|
143
|
+
call_vector_op(tensor, :logical_and, inputs[0], inputs[1], context, ->(t, u) { t && u })
|
144
144
|
end
|
145
145
|
|
146
|
-
register_op(:equal) do |context,
|
147
|
-
call_vector_op(:equal, inputs[0], inputs[1], context, ->(t, u) { t == u })
|
146
|
+
register_op(:equal) do |context, tensor, inputs|
|
147
|
+
call_vector_op(tensor, :equal, inputs[0], inputs[1], context, ->(t, u) { t == u })
|
148
148
|
end
|
149
149
|
|
150
|
-
register_op(:not_equal) do |context,
|
151
|
-
call_vector_op(:not_equal, inputs[0], inputs[1], context, ->(t, u) { t != u })
|
150
|
+
register_op(:not_equal) do |context, tensor, inputs|
|
151
|
+
call_vector_op(tensor, :not_equal, inputs[0], inputs[1], context, ->(t, u) { t != u })
|
152
152
|
end
|
153
153
|
|
154
154
|
register_op :index, no_eval: true do |_context, _tensor, inputs|
|
@@ -179,28 +179,80 @@ module TensorStream
|
|
179
179
|
end
|
180
180
|
end
|
181
181
|
|
182
|
-
register_op %i[flow_dynamic_stitch dynamic_stitch]
|
182
|
+
register_op %i[flow_dynamic_stitch dynamic_stitch] do |_context, _tensor, inputs|
|
183
183
|
indexes, data = inputs
|
184
184
|
merged = []
|
185
185
|
merge_dynamic_stitch(merged, indexes, data)
|
186
186
|
merged
|
187
187
|
end
|
188
188
|
|
189
|
+
register_op :gather do |_context, _tensor, inputs|
|
190
|
+
params, indexes = inputs
|
191
|
+
gather(params, indexes)
|
192
|
+
end
|
193
|
+
|
194
|
+
register_op :setdiff1d do |_context, tensor, inputs|
|
195
|
+
input, remove = inputs
|
196
|
+
idx = []
|
197
|
+
out = []
|
198
|
+
input.each_with_index do |x, index|
|
199
|
+
next if remove.include?(x)
|
200
|
+
out << x
|
201
|
+
idx << index
|
202
|
+
end
|
203
|
+
idx = idx.map { |i| Tensor.cast_dtype(i, tensor.options[:index_dtype]) } unless tensor.options[:index_dtype] == :int32
|
204
|
+
OutputGroup.new([out, idx], tensor.inputs.map(&:data_type))
|
205
|
+
end
|
206
|
+
|
207
|
+
register_op :cumprod do |context, tensor, inputs|
|
208
|
+
x = inputs[0]
|
209
|
+
c = fp_type?(tensor.data_type) ? 1.0 : 1
|
210
|
+
reverse_option = tensor.options[:reverse]
|
211
|
+
exclusive = tensor.options[:exclusive]
|
212
|
+
|
213
|
+
func = lambda do |arr|
|
214
|
+
return c if arr.nil?
|
215
|
+
count = arr.size
|
216
|
+
|
217
|
+
|
218
|
+
arr = arr.reverse if reverse_option
|
219
|
+
arr = [1] + arr if exclusive
|
220
|
+
|
221
|
+
start_prod = arr[0]
|
222
|
+
mapped = arr[1...count].map do |v|
|
223
|
+
start_prod = vector_op(start_prod, v, ->(a, b) { a * b })
|
224
|
+
end
|
225
|
+
|
226
|
+
arr = [arr[0]] + mapped
|
227
|
+
reverse_option ? arr.reverse : arr
|
228
|
+
end
|
229
|
+
reduction(context, tensor, func)
|
230
|
+
end
|
231
|
+
|
232
|
+
register_op :invert_permutation do |_context, _tensor, inputs|
|
233
|
+
input = inputs[0]
|
234
|
+
output = input.dup
|
235
|
+
input.size.times.each do |index|
|
236
|
+
output[input[index]] = index
|
237
|
+
end unless input.nil?
|
238
|
+
output
|
239
|
+
end
|
240
|
+
|
189
241
|
register_op :size do |_context, tensor, inputs|
|
190
242
|
input = inputs[0]
|
191
243
|
Tensor.cast_dtype(input.flatten.size, tensor.options[:out_type])
|
192
244
|
end
|
193
245
|
|
194
|
-
register_op %i[neg negate], no_eval: true do |context,
|
195
|
-
call_vector_op(:negate, inputs[0], nil, context, ->(t, _u) { -t })
|
246
|
+
register_op %i[neg negate], no_eval: true do |context, tensor, inputs|
|
247
|
+
call_vector_op(tensor, :negate, inputs[0], nil, context, ->(t, _u) { -t })
|
196
248
|
end
|
197
249
|
|
198
|
-
register_op :add, no_eval: true do |context,
|
250
|
+
register_op :add, no_eval: true do |context, tensor, inputs|
|
199
251
|
a, b = inputs
|
200
|
-
call_vector_op(:add, a, b, context, ->(t, u) { t + u })
|
252
|
+
call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
|
201
253
|
end
|
202
254
|
|
203
|
-
register_op :add_n, no_eval: true do |context,
|
255
|
+
register_op :add_n, no_eval: true do |context, tensor, inputs|
|
204
256
|
if inputs.size == 1
|
205
257
|
complete_eval(inputs[0], context)
|
206
258
|
elsif inputs.size > 1
|
@@ -208,48 +260,48 @@ module TensorStream
|
|
208
260
|
a = inputs.pop
|
209
261
|
until inputs.empty?
|
210
262
|
b = inputs.pop
|
211
|
-
a = call_vector_op(:add, a, b, context, ->(t, u) { t + u })
|
263
|
+
a = call_vector_op(tensor, :add, a, b, context, ->(t, u) { t + u })
|
212
264
|
end
|
213
265
|
a
|
214
266
|
end
|
215
267
|
end
|
216
268
|
|
217
|
-
register_op :sub, no_eval: true do |context,
|
269
|
+
register_op :sub, no_eval: true do |context, tensor, inputs|
|
218
270
|
a, b = inputs
|
219
|
-
call_vector_op(:sub, a, b, context, ->(t, u) { t - u })
|
271
|
+
call_vector_op(tensor, :sub, a, b, context, ->(t, u) { t - u })
|
220
272
|
end
|
221
273
|
|
222
|
-
register_op %i[floor_mod mod], no_eval: true do |context,
|
274
|
+
register_op %i[floor_mod mod], no_eval: true do |context, tensor, inputs|
|
223
275
|
a, b = inputs
|
224
|
-
call_vector_op(:
|
276
|
+
call_vector_op(tensor, :mod, a, b, context, ->(t, u) { t % u })
|
225
277
|
end
|
226
278
|
|
227
|
-
register_op %i[floor_div
|
279
|
+
register_op %i[floor_div], no_eval: true do |context, tensor, inputs|
|
228
280
|
a, b = inputs
|
229
281
|
if fp_type?(tensor.data_type)
|
230
|
-
call_vector_op(:
|
282
|
+
call_vector_op(tensor, :div, a, b, context, ->(t, u) { (t / u).to_i.to_f })
|
231
283
|
else
|
232
|
-
call_vector_op(:
|
284
|
+
call_vector_op(tensor, :div, a, b, context, ->(t, u) { t / u })
|
233
285
|
end
|
234
286
|
end
|
235
287
|
|
236
|
-
register_op :mul, no_eval: true do |context,
|
288
|
+
register_op :mul, no_eval: true do |context, tensor, inputs|
|
237
289
|
a, b = inputs
|
238
|
-
call_vector_op(:mul, a, b, context, ->(t, u) { t * u })
|
290
|
+
call_vector_op(tensor, :mul, a, b, context, ->(t, u) { t * u })
|
239
291
|
end
|
240
292
|
|
241
|
-
register_op :pow, no_eval: true do |context,
|
293
|
+
register_op :pow, no_eval: true do |context, tensor, inputs|
|
242
294
|
a, b = inputs
|
243
|
-
call_vector_op(:pow, a, b, context, ->(t, u) { t**u })
|
295
|
+
call_vector_op(tensor, :pow, a, b, context, ->(t, u) { t**u })
|
244
296
|
end
|
245
297
|
|
246
|
-
register_op :squared_difference, no_eval: true do |context,
|
298
|
+
register_op :squared_difference, no_eval: true do |context, tensor, inputs|
|
247
299
|
a, b = inputs
|
248
|
-
call_vector_op(:squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
|
300
|
+
call_vector_op(tensor, :squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
|
249
301
|
end
|
250
302
|
|
251
303
|
register_op %i[concat concat_v2] do |_context, tensor, inputs|
|
252
|
-
concat_array(inputs
|
304
|
+
concat_array(inputs, tensor.options[:axis])
|
253
305
|
end
|
254
306
|
|
255
307
|
register_op :round, no_eval: true do |context, _tensor, inputs|
|
@@ -268,6 +320,10 @@ module TensorStream
|
|
268
320
|
call_op(:tan, inputs[0], context, ->(t, _b) { Math.tan(t) })
|
269
321
|
end
|
270
322
|
|
323
|
+
register_op :atan, no_eval: true do |context, _tensor, inputs|
|
324
|
+
call_op(:atan, inputs[0], context, ->(t, _b) { Math.atan(t) })
|
325
|
+
end
|
326
|
+
|
271
327
|
register_op :sec, no_eval: true do |context, _tensor, inputs|
|
272
328
|
call_op(:sec, inputs[0], context, ->(t, _b) { Math.sec(t) })
|
273
329
|
end
|
@@ -328,9 +384,9 @@ module TensorStream
|
|
328
384
|
inputs[0]
|
329
385
|
end
|
330
386
|
|
331
|
-
register_op :sigmoid_grad, no_eval: true do |context,
|
387
|
+
register_op :sigmoid_grad, no_eval: true do |context, tensor, inputs|
|
332
388
|
a, b = inputs
|
333
|
-
call_vector_op(:sigmoid_grad, a, b, context, ->(t, u) { u * sigmoid(t) * (1 - sigmoid(t)) })
|
389
|
+
call_vector_op(tensor, :sigmoid_grad, a, b, context, ->(t, u) { u * sigmoid(t) * (1 - sigmoid(t)) })
|
334
390
|
end
|
335
391
|
|
336
392
|
register_op :random_uniform, no_eval: true do |_context, tensor, _inputs|
|
@@ -378,20 +434,72 @@ module TensorStream
|
|
378
434
|
|
379
435
|
register_op :assign, noop: true do |context, tensor, _inputs|
|
380
436
|
assign = tensor.inputs[0] || tensor
|
381
|
-
assign.value =
|
437
|
+
assign.value = global_eval(tensor, tensor.inputs[1], context)
|
382
438
|
assign.value
|
383
439
|
end
|
384
440
|
|
385
441
|
register_op :assign_add, noop: true do |context, tensor, _inputs|
|
386
|
-
tensor.inputs[0].value = process_vector_math_op(tensor.inputs[0], tensor.inputs[1], context, ->(t, u) { t + u })
|
442
|
+
tensor.inputs[0].value = process_vector_math_op(tensor, tensor.inputs[0], tensor.inputs[1], context, ->(t, u) { t + u })
|
443
|
+
tensor.inputs[0].value
|
444
|
+
end
|
445
|
+
|
446
|
+
register_op :variable, noop: true do |_context, tensor, _inputs|
|
387
447
|
tensor.inputs[0].value
|
388
448
|
end
|
389
449
|
|
390
450
|
register_op :assign_sub, noop: true do |context, tensor, _inputs|
|
391
|
-
tensor.inputs[0].value = process_vector_math_op(tensor.inputs[0], tensor.inputs[1], context, ->(t, u) { t - u })
|
451
|
+
tensor.inputs[0].value = process_vector_math_op(tensor, tensor.inputs[0], tensor.inputs[1], context, ->(t, u) { t - u })
|
392
452
|
tensor.inputs[0].value
|
393
453
|
end
|
394
454
|
|
455
|
+
register_op :stack do |_context, tensor, inputs|
|
456
|
+
axis = tensor.options[:axis] || 0
|
457
|
+
shape = shape_eval(inputs[0])
|
458
|
+
rank = shape.size + 1
|
459
|
+
elem_size = shape.empty? ? 1 : shape.reduce(:*)
|
460
|
+
output_buffer = Array.new(inputs.size * elem_size) { 0 }
|
461
|
+
new_shape = [inputs.size]
|
462
|
+
shape.inject(new_shape) { |ns, s| ns << s }
|
463
|
+
|
464
|
+
divisors = new_shape.dup.drop(1).reverse.inject([1]) do |a, s|
|
465
|
+
a << s * a.last
|
466
|
+
end.reverse
|
467
|
+
|
468
|
+
axis = rank + axis if axis < 0
|
469
|
+
rotated_shape = Array.new(axis + 1) { new_shape.shift }
|
470
|
+
new_shape = rotated_shape.rotate! + new_shape
|
471
|
+
|
472
|
+
multipliers = new_shape.dup.drop(1).reverse.inject([1]) do |a, s|
|
473
|
+
a << s * a.last
|
474
|
+
end.reverse
|
475
|
+
|
476
|
+
inputs.each_with_index do |input, index|
|
477
|
+
raw_input = input.is_a?(Array) ? input.flatten : [input]
|
478
|
+
start = index * divisors.first
|
479
|
+
|
480
|
+
raw_input.each_with_index do |x, index2|
|
481
|
+
index_map = []
|
482
|
+
ptr = start + index2
|
483
|
+
divisors.each_with_object(index_map) do |div, a|
|
484
|
+
a << (ptr / div.to_f).floor
|
485
|
+
ptr = ptr % div
|
486
|
+
end
|
487
|
+
|
488
|
+
rotated_index = Array.new(axis + 1) { index_map.shift }
|
489
|
+
index_map = rotated_index.rotate! + index_map
|
490
|
+
|
491
|
+
ptr2 = 0
|
492
|
+
multipliers.each_with_index do |m, idx|
|
493
|
+
ptr2 += index_map[idx] * m
|
494
|
+
end
|
495
|
+
|
496
|
+
output_buffer[ptr2] = x
|
497
|
+
end
|
498
|
+
end
|
499
|
+
|
500
|
+
TensorShape.reshape(output_buffer, new_shape)
|
501
|
+
end
|
502
|
+
|
395
503
|
register_op :mean, noop: true do |context, tensor, _inputs|
|
396
504
|
c = fp_type?(tensor.data_type) ? 0.0 : 0
|
397
505
|
func = lambda do |arr|
|
@@ -473,8 +581,20 @@ module TensorStream
|
|
473
581
|
call_op(:tanh_grad, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
|
474
582
|
end
|
475
583
|
|
476
|
-
register_op :transpose do |_context,
|
477
|
-
inputs[0]
|
584
|
+
register_op :transpose do |_context, tensor, inputs|
|
585
|
+
shape = shape_eval(inputs[0])
|
586
|
+
rank = get_rank(inputs[0])
|
587
|
+
perm = inputs[1] || (0...rank).to_a.reverse
|
588
|
+
if rank == 2 && perm.nil? # use native transpose for general case
|
589
|
+
inputs[0].transpose
|
590
|
+
else
|
591
|
+
arr = inputs[0].flatten
|
592
|
+
|
593
|
+
new_shape = perm.map { |p| shape[p] }
|
594
|
+
new_arr = Array.new(shape.reduce(:*)) { 0 }
|
595
|
+
transpose_with_perm(arr, new_arr, shape, new_shape, perm)
|
596
|
+
TensorShape.reshape(new_arr, new_shape)
|
597
|
+
end
|
478
598
|
end
|
479
599
|
|
480
600
|
register_op :eye do |_context, tensor, inputs|
|
@@ -491,7 +611,7 @@ module TensorStream
|
|
491
611
|
end
|
492
612
|
end
|
493
613
|
|
494
|
-
register_op :expand_dims do |
|
614
|
+
register_op :expand_dims do |_context, _tensor, inputs|
|
495
615
|
val, axis = inputs
|
496
616
|
axis = axis.nil? ? 0 : axis
|
497
617
|
|
@@ -504,12 +624,12 @@ module TensorStream
|
|
504
624
|
end
|
505
625
|
|
506
626
|
register_op :cond, noop: true do |context, tensor, inputs|
|
507
|
-
pred =
|
627
|
+
pred = global_eval(tensor, tensor.options[:pred], context)
|
508
628
|
|
509
629
|
if all_true?(pred)
|
510
|
-
|
630
|
+
global_eval(tensor, inputs[0], context)
|
511
631
|
else
|
512
|
-
|
632
|
+
global_eval(tensor, inputs[1], context)
|
513
633
|
end
|
514
634
|
end
|
515
635
|
|
@@ -518,24 +638,24 @@ module TensorStream
|
|
518
638
|
call_3way_vector_op(pred, inputs[0], inputs[1], context, ->(t, u, v) { t ? u : v })
|
519
639
|
end
|
520
640
|
|
521
|
-
register_op :less do |context,
|
641
|
+
register_op :less do |context, tensor, inputs|
|
522
642
|
a, b = inputs
|
523
|
-
call_vector_op(:less, a, b, context, ->(t, u) { t < u })
|
643
|
+
call_vector_op(tensor, :less, a, b, context, ->(t, u) { t < u })
|
524
644
|
end
|
525
645
|
|
526
|
-
register_op :greater do |context,
|
646
|
+
register_op :greater do |context, tensor, inputs|
|
527
647
|
a, b = inputs
|
528
|
-
call_vector_op(:greater, a, b, context, ->(t, u) { t > u })
|
648
|
+
call_vector_op(tensor, :greater, a, b, context, ->(t, u) { t > u })
|
529
649
|
end
|
530
650
|
|
531
|
-
register_op :greater_equal do |context,
|
651
|
+
register_op :greater_equal do |context, tensor, inputs|
|
532
652
|
a, b = inputs
|
533
|
-
call_vector_op(:greater_equal, a, b, context, ->(t, u) { t >= u })
|
653
|
+
call_vector_op(tensor, :greater_equal, a, b, context, ->(t, u) { t >= u })
|
534
654
|
end
|
535
655
|
|
536
|
-
register_op :less_equal do |context,
|
656
|
+
register_op :less_equal do |context, tensor, inputs|
|
537
657
|
a, b = inputs
|
538
|
-
call_vector_op(:greater_equal, a, b, context, ->(t, u) { t <= u })
|
658
|
+
call_vector_op(tensor, :greater_equal, a, b, context, ->(t, u) { t <= u })
|
539
659
|
end
|
540
660
|
|
541
661
|
register_op :fill do |_context, _tensor, inputs|
|
@@ -622,8 +742,8 @@ module TensorStream
|
|
622
742
|
get_rank(inputs[0])
|
623
743
|
end
|
624
744
|
|
625
|
-
register_op
|
626
|
-
process_vector_math_op(inputs[0], inputs[1], context, ->(t, u) { t / u })
|
745
|
+
register_op %i[div real_div], noop: true do |context, tensor, inputs|
|
746
|
+
process_vector_math_op(tensor, inputs[0], inputs[1], context, ->(t, u) { t / u })
|
627
747
|
end
|
628
748
|
|
629
749
|
register_op :reshape do |_context, _tensor, inputs|
|
@@ -646,31 +766,44 @@ module TensorStream
|
|
646
766
|
arr_pad(inputs[0], p, tensor.data_type)
|
647
767
|
end
|
648
768
|
|
649
|
-
register_op %i[max maximum], noop: true do |context,
|
650
|
-
call_vector_op(:max, inputs[0], inputs[1], context, ->(t, u) { [t, u].max })
|
769
|
+
register_op %i[max maximum], noop: true do |context, tensor, inputs|
|
770
|
+
call_vector_op(tensor, :max, inputs[0], inputs[1], context, ->(t, u) { [t, u].max })
|
651
771
|
end
|
652
772
|
|
653
|
-
register_op %i[min minimum], noop: true do |context,
|
654
|
-
call_vector_op(:min, inputs[0], inputs[1], context, ->(t, u) { [t, u].min })
|
773
|
+
register_op %i[min minimum], noop: true do |context, tensor, inputs|
|
774
|
+
call_vector_op(tensor, :min, inputs[0], inputs[1], context, ->(t, u) { [t, u].min })
|
655
775
|
end
|
656
776
|
|
657
777
|
register_op :apply_gradient_descent do |context, tensor, inputs|
|
658
778
|
target_var, learning_rate, delta = inputs
|
659
779
|
assign = tensor.inputs[0] || tensor
|
660
780
|
|
661
|
-
assign.value = process_vector_math_op(target_var, delta, context, ->(t, u) { t - u * learning_rate })
|
781
|
+
assign.value = process_vector_math_op(tensor, target_var, delta, context, ->(t, u) { t - u * learning_rate })
|
782
|
+
assign.value
|
783
|
+
end
|
784
|
+
|
785
|
+
register_op :apply_momentum do |context, tensor, inputs|
|
786
|
+
target_var, momentum_var, learning_rate, grad, momentum = inputs
|
787
|
+
assign = tensor.inputs[0] || tensor
|
788
|
+
assign_acc = tensor.inputs[1]
|
789
|
+
assign_acc.value = process_vector_math_op(tensor, momentum_var, grad, context, ->(t, u) { t * momentum + u })
|
790
|
+
if tensor.options[:use_nesterov]
|
791
|
+
delta = process_vector_math_op(tensor, grad, momentum_var, context, ->(g, acc) { g * learning_rate + acc * momentum * learning_rate })
|
792
|
+
assign.value = process_vector_math_op(tensor,target_var, delta, context, ->(t, u) { t - u })
|
793
|
+
else
|
794
|
+
assign.value = process_vector_math_op(tensor, target_var, momentum_var, context, ->(v, acc) { v - acc * learning_rate })
|
795
|
+
end
|
662
796
|
assign.value
|
663
797
|
end
|
664
798
|
|
665
|
-
register_op :broadcast_gradient_args do |_context,
|
799
|
+
register_op :broadcast_gradient_args do |_context, tensor, inputs|
|
666
800
|
rx, ry = get_broadcast_gradient_args(inputs[0], inputs[1])
|
667
|
-
OutputGroup.new([rx, ry])
|
801
|
+
OutputGroup.new([rx, ry], tensor.inputs.map(&:data_type))
|
668
802
|
end
|
669
803
|
|
670
804
|
register_op :tile do |_context, _tensor, inputs|
|
671
805
|
input, multiples = inputs
|
672
806
|
rank = get_rank(input)
|
673
|
-
|
674
807
|
raise '1D or higher tensor required' if rank.zero?
|
675
808
|
raise "invalid multiple size passed #{rank} != #{multiples.size}" if rank != multiples.size
|
676
809
|
|
@@ -678,8 +811,8 @@ module TensorStream
|
|
678
811
|
tile.nil? ? [] : tile
|
679
812
|
end
|
680
813
|
|
681
|
-
register_op :flow_group, noop: true do |context,
|
682
|
-
inputs.collect { |input|
|
814
|
+
register_op :flow_group, noop: true do |context, tensor, inputs|
|
815
|
+
inputs.collect { |input| global_eval(tensor, input, context) }
|
683
816
|
end
|
684
817
|
|
685
818
|
register_op :softmax do |_context, _tensor, inputs|
|
@@ -687,11 +820,11 @@ module TensorStream
|
|
687
820
|
end
|
688
821
|
|
689
822
|
register_op :save_v2 do |context, tensor, inputs|
|
690
|
-
prefix, tensor_names, shape_and_slices = inputs[0..3]
|
823
|
+
# prefix, tensor_names, shape_and_slices = inputs[0..3]
|
691
824
|
end
|
692
825
|
|
693
826
|
register_op :restore_v2 do |context, tensor, inputs|
|
694
|
-
prefix, tensor_names, shape_and_slices = inputs[0..3]
|
827
|
+
# prefix, tensor_names, shape_and_slices = inputs[0..3]
|
695
828
|
end
|
696
829
|
|
697
830
|
register_op :softmax_grad do |_context, _tensor, inputs|
|
@@ -744,24 +877,36 @@ module TensorStream
|
|
744
877
|
end
|
745
878
|
end
|
746
879
|
|
747
|
-
register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context,
|
880
|
+
register_op %i[softmax_cross_entropy_with_logits_v2 softmax_cross_entropy_with_logits] do |_context, tensor, inputs|
|
748
881
|
last_dimen_list = last_axis(inputs[0])
|
749
882
|
input_shape = shape_eval(inputs[0])
|
883
|
+
rank = input_shape.size - 1
|
750
884
|
labels = last_axis(inputs[1])
|
751
885
|
func = lambda { |logits, label|
|
752
886
|
c = logits.max
|
753
887
|
transformed_logits = logits.map { |l| l - c }
|
754
888
|
sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
|
755
|
-
transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
|
889
|
+
losses = transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
|
890
|
+
probs = transformed_logits.zip(label).map { |x, y| (Math.exp(x) / sum) - y }
|
891
|
+
[losses, probs]
|
756
892
|
}
|
757
893
|
|
758
894
|
if input_shape.size == 1
|
759
|
-
func.call(last_dimen_list, labels)
|
895
|
+
loss, prob = func.call(last_dimen_list, labels)
|
896
|
+
loss = reduce(loss, rank, false)
|
897
|
+
OutputGroup.new([loss, prob], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
|
760
898
|
else
|
761
|
-
|
762
|
-
|
899
|
+
losses = []
|
900
|
+
backprobs = []
|
901
|
+
arr = last_dimen_list.zip(labels).each do |list, label|
|
902
|
+
loss, prob = func.call(list, label)
|
903
|
+
losses << loss
|
904
|
+
backprobs << prob
|
763
905
|
end
|
764
|
-
TensorShape.reshape(
|
906
|
+
reshaped_losses = TensorShape.reshape(losses.flatten, input_shape)
|
907
|
+
reshaped_backprops = TensorShape.reshape(backprobs.flatten, input_shape)
|
908
|
+
reshaped_losses = reduce(reshaped_losses, rank, false)
|
909
|
+
OutputGroup.new([reshaped_losses, reshaped_backprops], [tensor.inputs[0].data_type, tensor.inputs[0].data_type])
|
765
910
|
end
|
766
911
|
end
|
767
912
|
|
@@ -791,7 +936,7 @@ module TensorStream
|
|
791
936
|
register_op :check_numerics do |context, tensor, inputs|
|
792
937
|
message = tensor.options[:message]
|
793
938
|
f = lambda { |t, _b|
|
794
|
-
raise "#{message} Invalid argument" if t.nan? || t.infinite?
|
939
|
+
raise TensorStream::InvalidArgumentError, "#{message} Invalid argument" if t.nan? || t.infinite?
|
795
940
|
t
|
796
941
|
}
|
797
942
|
call_op(:check_numerics, inputs[0], context, f)
|
@@ -799,13 +944,9 @@ module TensorStream
|
|
799
944
|
|
800
945
|
def eval_operation(tensor, child_context)
|
801
946
|
return @context[tensor.name] if @context.key?(tensor.name)
|
947
|
+
# puts "ruby: #{tensor.name}"
|
802
948
|
invoke(tensor, child_context).tap do |result|
|
803
|
-
|
804
|
-
# if tensor.name.start_with?('softmax_cross_entropy_with_logits')
|
805
|
-
# puts result.inspect
|
806
|
-
# end
|
807
|
-
# result.flatten.each do |a|
|
808
|
-
# end if result.is_a?(Array)
|
949
|
+
|
809
950
|
if tensor.breakpoint
|
810
951
|
a = resolve_placeholder(tensor.inputs[0], child_context) if tensor.inputs && tensor.inputs[0]
|
811
952
|
b = resolve_placeholder(tensor.inputs[1], child_context) if tensor.inputs && tensor.inputs[1]
|
@@ -826,20 +967,21 @@ module TensorStream
|
|
826
967
|
@context[tensor.name] = result
|
827
968
|
end
|
828
969
|
rescue EvaluatorExcecutionException => e
|
829
|
-
raise e
|
970
|
+
raise e, "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true, 1)} defined at #{tensor.source}"
|
830
971
|
rescue TensorStreamError => e
|
831
|
-
raise e
|
972
|
+
raise e, "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true, 1)} defined at #{tensor.source}"
|
832
973
|
rescue StandardError => e
|
833
|
-
|
834
|
-
|
974
|
+
|
975
|
+
a = resolve_placeholder(tensor.inputs[0], child_context) if tensor.inputs && tensor.inputs[0]
|
976
|
+
b = resolve_placeholder(tensor.inputs[1], child_context) if tensor.inputs && tensor.inputs[1]
|
835
977
|
puts e.message
|
836
978
|
puts e.backtrace.join("\n")
|
837
979
|
# shape_a = a.shape.shape if a
|
838
980
|
# shape_b = b.shape.shape if b
|
839
981
|
# dtype_a = a.data_type if a
|
840
982
|
# dtype_b = b.data_type if b
|
841
|
-
|
842
|
-
|
983
|
+
a = complete_eval(a, child_context)
|
984
|
+
b = complete_eval(b, child_context)
|
843
985
|
# puts "name: #{tensor.given_name}"
|
844
986
|
# # puts "op: #{tensor.to_math(true, 1)}"
|
845
987
|
# puts "A #{shape_a} #{dtype_a}: #{a}" if a
|
@@ -909,23 +1051,10 @@ module TensorStream
|
|
909
1051
|
end
|
910
1052
|
|
911
1053
|
def reduction(child_context, tensor, func)
|
912
|
-
val =
|
913
|
-
axis =
|
914
|
-
keep_dims =
|
915
|
-
|
916
|
-
return val if axis && axis.is_a?(Array) && axis.empty?
|
917
|
-
|
918
|
-
axis = if axis.nil?
|
919
|
-
nil
|
920
|
-
elsif axis.is_a?(Array)
|
921
|
-
return val if axis.empty?
|
922
|
-
|
923
|
-
axis.map { |a| a < 0 ? rank - a.abs : a }
|
924
|
-
else
|
925
|
-
axis < 0 ? rank - axis.abs : axis
|
926
|
-
end
|
927
|
-
|
928
|
-
reduce_axis(0, axis, val, keep_dims, func)
|
1054
|
+
val = global_eval(tensor, tensor.inputs[0], child_context)
|
1055
|
+
axis = global_eval(tensor, tensor.inputs[1], child_context)
|
1056
|
+
keep_dims = global_eval(tensor, tensor.options[:keepdims], child_context)
|
1057
|
+
reduce(val, axis, keep_dims, func)
|
929
1058
|
end
|
930
1059
|
|
931
1060
|
def arr_pad(arr, paddings, data_type = :float32, rank = 0)
|
@@ -960,15 +1089,15 @@ module TensorStream
|
|
960
1089
|
TensorStream.send(op.to_sym, a)
|
961
1090
|
end
|
962
1091
|
|
963
|
-
def call_vector_op(op, a, b, child_context, func)
|
964
|
-
process_vector_math_op(a, b, child_context, func)
|
1092
|
+
def call_vector_op(tensor, op, a, b, child_context, func)
|
1093
|
+
process_vector_math_op(tensor, a, b, child_context, func)
|
965
1094
|
rescue FullEvalNotPossible
|
966
1095
|
TensorStream.send(op.to_sym, a, b)
|
967
1096
|
end
|
968
1097
|
|
969
|
-
def process_vector_math_op(a, b, child_context, op)
|
970
|
-
eval_a =
|
971
|
-
eval_b =
|
1098
|
+
def process_vector_math_op(tensor, a, b, child_context, op)
|
1099
|
+
eval_a = global_eval(tensor, a, child_context) unless a.nil?
|
1100
|
+
eval_b = global_eval(tensor, b, child_context) unless b.nil?
|
972
1101
|
|
973
1102
|
raise FullEvalNotPossible.new, "full eval not possible for #{a.name}" if eval_a.is_a?(Tensor) || eval_b.is_a?(Tensor)
|
974
1103
|
|
@@ -1025,28 +1154,6 @@ module TensorStream
|
|
1025
1154
|
Tensor.cast_dtype(var, placeholder.data_type)
|
1026
1155
|
end
|
1027
1156
|
|
1028
|
-
def reduce_axis(current_axis, axis, val, keep_dims, f = ->(a, b) { a + b })
|
1029
|
-
return val unless val.is_a?(Array)
|
1030
|
-
|
1031
|
-
r = val.collect do |v|
|
1032
|
-
reduce_axis(current_axis + 1, axis, v, keep_dims, f)
|
1033
|
-
end
|
1034
|
-
|
1035
|
-
should_reduce_axis = axis.nil? || (axis.is_a?(Array) && axis.include?(current_axis)) || (current_axis == axis)
|
1036
|
-
|
1037
|
-
if should_reduce_axis
|
1038
|
-
reduced_val = r[0]
|
1039
|
-
if r.size > 1
|
1040
|
-
reduced_val = f.call(r[0..val.size])
|
1041
|
-
elsif r.empty?
|
1042
|
-
reduced_val = f.call(nil)
|
1043
|
-
end
|
1044
|
-
keep_dims ? [reduced_val] : reduced_val
|
1045
|
-
else
|
1046
|
-
r
|
1047
|
-
end
|
1048
|
-
end
|
1049
|
-
|
1050
1157
|
# handle 3 tensor math operations
|
1051
1158
|
def call_3way_vector_op(v_a, v_b, v_c, child_context, op = ->(a, b, c) { a + b + c })
|
1052
1159
|
return op.call(v_a, v_b, v_c) unless v_a.is_a?(Array)
|