tensor_stream 1.0.0 → 1.0.1

data/lib/tensor_stream/helpers/op_helper.rb

@@ -1,4 +1,3 @@
-
 module TensorStream
   # module that contains helper functions useful for ops
   module OpHelper
@@ -7,7 +6,7 @@ module TensorStream
 
       op = default_graph.add_op!(code.to_sym, *args)
       if !default_graph.get_dependency_scope.nil?
-        i_op(:identity, op, default_graph.get_dependency_scope, name: [op.name, 'tuple', 'control_dependency'].join('/'))
+        i_op(:identity, op, default_graph.get_dependency_scope, name: [op.name, "tuple", "control_dependency"].join("/"))
       else
         op
       end
@@ -16,10 +15,10 @@ module TensorStream
     # same as op but with a marker that it was internal generated
     def i_op(code, *args)
       options = if args.last.is_a?(Hash)
-                  args.pop
-                else
-                  {}
-                end
+        args.pop
+      else
+        {}
+      end
 
       args << options.merge(internal: true)
       Graph.get_default_graph.add_op!(code.to_sym, *args)
@@ -65,8 +64,8 @@ module TensorStream
     end
 
     def format_source(trace)
-      grad_source = trace.detect { |c| c.to_s.include?(File.join('lib', 'tensor_stream', 'math_gradients')) }
-      source = trace.reject { |c| c.to_s.include?(File.join('lib', 'tensor_stream')) }.first
+      grad_source = trace.detect { |c| c.to_s.include?(File.join("lib", "tensor_stream", "math_gradients")) }
+      source = trace.reject { |c| c.to_s.include?(File.join("lib", "tensor_stream")) }.first
       [grad_source, trace].compact.join("\n")
     end
 
@@ -94,7 +93,7 @@ module TensorStream
       axes_shape = i_op(:shape, axes)
 
       TensorStream.dynamic_stitch([TensorStream.range(0, input_rank), axes],
-                                  [input_shape, i_op(:fill, axes_shape, 1)])
+        [input_shape, i_op(:fill, axes_shape, 1)])
     end
   end
 end

data/lib/tensor_stream/helpers/string_helper.rb

@@ -4,28 +4,28 @@ module TensorStream
   module StringHelper
     def camelize(string, uppercase_first_letter = true)
       string = if uppercase_first_letter
-                 string.sub(/^[a-z\d]*/) { $&.capitalize }
-               else
-                 string.sub(/^(?:(?=\b|[A-Z_])|\w)/) { $&.downcase }
-               end
-      string.gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }.gsub('/', '::')
+        string.sub(/^[a-z\d]*/) { $&.capitalize }
+      else
+        string.sub(/^(?:(?=\b|[A-Z_])|\w)/) { $&.downcase }
+      end
+      string.gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }.gsub("/", "::")
     end
 
     def underscore(string)
-      string.gsub(/::/, '/')
-            .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
-            .gsub(/([a-z\d])([A-Z])/, '\1_\2')
-            .tr("-", "_").downcase
+      string.gsub(/::/, "/").
+        gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').
+        gsub(/([a-z\d])([A-Z])/, '\1_\2').
+        tr("-", "_").downcase
     end
 
     def symbolize_keys(hash)
-      hash.map do |k, v|
+      hash.map { |k, v|
         [k.to_sym, v]
-      end.to_h
+      }.to_h
     end
 
     def constantize(camel_cased_word)
-      names = camel_cased_word.split('::')
+      names = camel_cased_word.split("::")
 
       # Trigger a built-in NameError exception including the ill-formed constant in the message.
       Object.const_get(camel_cased_word) if names.empty?
@@ -43,11 +43,11 @@ module TensorStream
 
           # Go down the ancestors to check if it is owned directly. The check
           # stops when we reach Object or the end of ancestors tree.
-          constant = constant.ancestors.inject do |const, ancestor|
+          constant = constant.ancestors.inject { |const, ancestor|
             break const    if ancestor == Object
             break ancestor if ancestor.const_defined?(name, false)
             const
-          end
+          }
 
           # owner is in Object, so raise
           constant.const_get(name, false)
@@ -55,4 +55,4 @@ module TensorStream
       end
     end
   end
-end
+end

data/lib/tensor_stream/helpers/tensor_mixins.rb

@@ -1,7 +1,7 @@
 module TensorStream
   module TensorMixins
     def +(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:add, self, other)
     end
 
@@ -10,22 +10,22 @@ module TensorStream
     end
 
     def *(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:mul, self, TensorStream.convert_to_tensor(other, dtype: data_type))
     end
 
     def **(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:pow, self, TensorStream.convert_to_tensor(other, dtype: data_type))
     end
 
     def /(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:div, self, TensorStream.convert_to_tensor(other, dtype: data_type))
     end
 
     def -(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:sub, self, TensorStream.convert_to_tensor(other, dtype: data_type))
     end
 
@@ -58,51 +58,51 @@ module TensorStream
     end
 
     def zero?
-      _op(:equal, self, TensorStream.constant(0, dtype: data_type, name: 'equal/is_zero?'))
+      _op(:equal, self, TensorStream.constant(0, dtype: data_type, name: "equal/is_zero?"))
     end
 
     def ==(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:equal, self, other)
     end
 
     def <(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:less, self, other)
     end
 
     def !=(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:not_equal, self, other)
     end
 
     def >(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:greater, self, other)
     end
 
     def >=(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:greater_equal, self, other)
     end
 
     def <=(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:less_equal, self, other)
     end
 
     def and(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:logical_and, self, other)
     end
 
     def matmul(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:mat_mul, self, other)
     end
 
     def dot(other)
-      _a, other = TensorStream.check_data_types(self, other)
+      TensorStream.check_data_types(self, other)
       _op(:mat_mul, self, other)
     end
 
@@ -126,10 +126,10 @@ module TensorStream
                     :mean
                   else
                     raise "unsupported reduce op type #{op_type} valid values are :+, :*, :prod, :mean"
-                  end
+      end
       raise "blocks are not supported for tensors" if block_given?
 
       TensorStream.reduce(reduce_op, self, axis, keepdims: keepdims, name: name)
     end
   end
-end
+end

data/lib/tensor_stream/images.rb

@@ -13,4 +13,4 @@ module TensorStream
       _op(:encode_png, contents, compression: compression, name: name, new_shape: new_shape, resample_method: resample_method)
     end
   end
-end
+end

data/lib/tensor_stream/initializer.rb

@@ -13,4 +13,4 @@ module TensorStream
       nil
     end
   end
-end
+end

data/lib/tensor_stream/math_gradients.rb

@@ -12,10 +12,10 @@ module TensorStream
       return i_op(:ones_like, tensor) if tensor.equal?(wrt_dx)
       return i_op(:zeros_like, wrt_dx) unless wrt_dx.consumers.include?(tensor.name)
 
-      nodes_to_compute = wrt_dx.consumers.select do |t|
+      nodes_to_compute = wrt_dx.consumers.select { |t|
         node = tensor.graph.nodes[t]
         node.consumers.include?(tensor.name) || node.equal?(tensor)
-      end.compact + [wrt_dx.name]
+      }.compact + [wrt_dx.name]
 
       grad = i_op(:fill, ts.shape(tensor), ts.constant(1, dtype: wrt_dx.data_type))
 
@@ -30,12 +30,12 @@ module TensorStream
       computed_op = _compute_derivative(tensor, grad)
 
       if computed_op.is_a?(Array)
-        grads = computed_op.each_with_index.collect do |op_grad, index|
+        grads = computed_op.each_with_index.collect { |op_grad, index|
           next if op_grad.nil?
           next unless nodes_to_compute.include?(tensor.inputs[index].name)
 
           _propagate(op_grad, tensor.inputs[index], stop_tensor, nodes_to_compute, stop_gradients)
-        end.compact
+        }.compact
 
         return nil if grads.empty?
         grads.size > 1 ? ts.add_n(grads) : grads[0]
@@ -48,7 +48,7 @@ module TensorStream
       end
     end
 
-    #TODO: refactor and implement registerGradient
+    # TODO: refactor and implement registerGradient
     def self._compute_derivative(node, grad)
       node.graph.name_scope("#{node.name}_grad") do
         x = node.inputs[0] if node.inputs[0]
@@ -58,14 +58,6 @@ module TensorStream
         case node.operation
         when :add_n
           return [grad] * node.inputs.size
-        when :add
-          return [grad, grad] if shapes_fully_specified_and_equal(x, y)
-          sx = ts.shape(x, name: 'add/shape_x')
-          sy = ts.shape(y, name: 'add/shape_y')
-          rx, ry = _broadcast_gradient_args(sx, sy)
-
-          [ts.reshape(ts.reduce_sum(grad, rx, name: 'add/reduce_sum_x'), sx),
-           ts.reshape(ts.reduce_sum(grad, ry, name: 'add/reduce_sum_y'), sy)]
         when :asin
           ts.control_dependencies([grad]) do
             x2 = ts.square(x)
@@ -89,75 +81,6 @@ module TensorStream
             inv = ts.reciprocal(ts.add(one, x2))
             grad * inv
           end
-        when :fill
-          [nil, ts.reduce_sum(grad)]
-        when :sub
-          return [grad, -grad] if shapes_fully_specified_and_equal(x, y)
-
-          sx = ts.shape(x, name: 'sub/shape_x')
-          sy = ts.shape(y, name: 'sub/shape_y')
-          rx, ry = _broadcast_gradient_args(sx, sy)
-
-          [ts.reshape(ts.reduce_sum(grad, rx, name: 'add/reduce_sub_x'), sx),
-           -ts.reshape(ts.reduce_sum(grad, ry, name: 'add/reduce_sub_y'), sy)]
-        when :mul
-          sx = ts.shape(x)
-          sy = ts.shape(y)
-          rx, ry = _broadcast_gradient_args(sx, sy)
-
-          [ts.reshape(ts.reduce_sum(ts.mul(grad, y), rx), sx),
-           ts.reshape(ts.reduce_sum(ts.mul(x, grad), ry), sy)]
-        when :div
-          sx = i_op(:shape, x)
-          sy = i_op(:shape, y)
-          rx, ry = _broadcast_gradient_args(sx, sy)
-
-          [ts.reshape(ts.reduce_sum(ts.div(grad, y), rx), sx),
-           ts.reshape(ts.reduce_sum(grad * ts.div(ts.div(-x, y), y), ry), sy)]
-        when :mod
-          sx = ts.shape(x)
-          sy = ts.shape(y)
-          rx, ry = _broadcast_gradient_args(sx, sy)
-          floor_xy = ts.floor_div(x, y)
-          gx = ts.reshape(ts.reduce_sum(grad, rx), sx)
-          gy = ts.reshape(ts.reduce_sum(grad * ts.negative(floor_xy), ry), sy)
-
-          [gx, gy]
-        when :prod
-          input_shape = ts.shape(x)
-          y = ts.range(0, ts.rank(x)) if y.nil?
-          reduction_indices = ts.reshape(y, [-1])
-
-          output_shape_kept_dims = ts.reduced_shape(input_shape, y)
-          tile_scaling = _safe_shape_div(input_shape, output_shape_kept_dims)
-          grad = ts.reshape(grad, output_shape_kept_dims)
-          grad = ts.tile(grad, tile_scaling)
-
-          perm, reduced_num, other_num = ts.device("/cpu:0") do
-            rank = ts.rank(x)
-            reduction_indices = (reduction_indices + rank) % rank
-            reduced = ts.cast(reduction_indices, :int32)
-            idx = ts.range(0, rank)
-            other, = ts.setdiff1d(idx, reduced)
-            [ts.concat([reduced, other], 0),
-             ts.reduce_prod(ts.gather(input_shape, reduced)),
-             ts.reduce_prod(ts.gather(input_shape, other))]
-          end
-
-          permuted = ts.transpose(x, perm)
-          permuted_shape = ts.shape(permuted)
-
-          reshaped = ts.reshape(permuted, [reduced_num, other_num])
-
-          # Calculate product, leaving out the current entry
-          left = ts.cumprod(reshaped, axis: 0, exclusive: true)
-          right = ts.cumprod(reshaped, axis: 0, exclusive: true, reverse: true)
-          y = ts.reshape(left * right, permuted_shape)
-
-          # Invert the transpose and reshape operations.
-          # Make sure to set the statically known shape information through a reshape.
-          out = grad * ts.transpose(y, ts.invert_permutation(perm))
-          [ts.reshape(out, input_shape, name: 'prod'), nil]
         when :squared_difference
           sx = i_op(:shape, x)
           sy = i_op(:shape, y)
@@ -166,63 +89,13 @@ module TensorStream
           x_grad = ts.mul(2.0, grad) * (x - y)
 
           [ts.reshape(ts.reduce_sum(x_grad, rx), sx),
-           ts.reshape(-ts.reduce_sum(x_grad, ry), sy)]
-        when :mat_mul
-          t_a = node.options[:transpose_a]
-          t_b = node.options[:transpose_b]
-
-          if !t_a && !t_b
-            grad_a = ts.matmul(grad, y, transpose_b: true)
-            grad_b = ts.matmul(x, grad, transpose_a: true)
-          elsif !ta && tb
-            grad_a = ts.matmul(grad, y)
-            grad_b = ts.matmul(grad, x, transpose_a: true)
-          elsif t_a && !t_b
-            grad_a = ts.matmul(y, grad, transpose_b: true)
-            grad_b = ts.matmul(x, grad)
-          elsif t_a && t_b
-            grad_a = ts.matmul(y, grad, transpose_a: true, transpose_b: true)
-            grad_b = ts.matmul(grad, x, transpose_a: true, transpose_b: true)
-          end
-
-          [grad_a, grad_b]
-        when :sin
-          grad * ts.cos(x)
-        when :tanh
-          grad * i_op(:tanh_grad, x)
-        when :pow
-          z = node
-          sx = ts.shape(x)
-          sy = ts.shape(y)
-          rx, ry = _broadcast_gradient_args(sx, sy)
-          gx = ts.reduce_sum(grad * y * ts.pow(x, y - 1), rx)
-
-          log_x = ts.where(x > 0, ts.log(x), ts.zeros_like(x))
-          gy = ts.reduce_sum(grad * z * log_x, ry)
-
-          [gx, gy]
+           ts.reshape(-ts.reduce_sum(x_grad, ry), sy),]
         when :abs
           grad * ts.sign(x)
-        when :log
-          grad * ts.reciprocal(x)
-        when :cos
-          -grad * ts.sin(x)
-        when :max
-          _min_or_max_grad(node.inputs, grad, ->(a, b) { ts.greater_equal(a, b) })
-        when :min
-          _min_or_max_grad(node.inputs, grad, ->(a, b) { ts.less_equal(a, b) })
-        when :tan
-          secx = ts.reciprocal(ts.cos(x))
-          secx2 = ts.square(secx)
-          grad * secx2
-        when :negate
-          -grad
         when :exp
           grad * node
         when :identity, :print
           grad
-        when :sign
-          ts.zeros(ts.shape(x), dtype: x.data_type)
         when :tile
           input_shape = ts.shape(x)
           split_shape = ts.reshape(ts.transpose(ts.stack([y, input_shape])), [-1])
@@ -230,8 +103,6 @@ module TensorStream
           input_grad = ts.reduce_sum(ts.reshape(grad, split_shape), axes)
 
           [input_grad, nil]
-        when :sum
-          _sum_grad(x, y, grad)
         when :reciprocal
           -grad * (ts.constant(1, dtype: x.dtype) / x**2)
         when :sqrt
@@ -245,14 +116,6 @@ module TensorStream
           x_mask = i_op(:where, x, i_op(:ones_like, y), i_op(:zeros_like, z))
           y_mask = i_op(:where, x, i_op(:zeros_like, y), i_op(:ones_like, z))
           [nil, x_mask * grad, y_mask * grad]
-        when :case
-          n_preds = node.inputs.size - 2
-
-          case_grads = Array.new(n_preds) do |index|
-            i_op(:case_grad, index, node.inputs[0], node.inputs[2 + index], grad)
-          end
-
-          [nil, i_op(:case_grad, -1, node.inputs[0], node.inputs[1], grad)] + case_grads
         when :mean
           sum_grad = _sum_grad(x, y, grad)[0]
           input_shape = ts.shape(x)
@@ -261,8 +124,6 @@ module TensorStream
           [ts.div(sum_grad, ts.cast(factor, sum_grad.data_type)), nil]
         when :log1p
           grad * ts.reciprocal(i_cons(1, dtype: grad.data_type) + x)
-        when :sigmoid
-          i_op(:sigmoid_grad, x, grad)
         when :sigmoid_grad
           gb = grad * y
           [gb - 2.0 * gb * x, i_op(:sigmoid_grad, x, grad)]
@@ -275,15 +136,9 @@ module TensorStream
         when :sparse_softmax_cross_entropy_with_logits
           output = node
           [_broadcast_mul(grad, output[1]), nil]
-        when :floor, :ceil, :round
-          # non differentiable
-          nil
-        when :zeros_like
+         when :zeros_like
           # non differentiable
           nil
-        when :argmin, :argmax, :floor_div
-          # non differentiable
-          [nil, nil]
         when :transpose
           return [ts.transpose(grad, ts.invert_permutation(y)), nil]
         when :index
@@ -294,19 +149,15 @@ module TensorStream
             multiplier = node.inputs[0].shape.shape[0]
             filler = ts.zeros_like(grad)
 
-            res = Array.new(multiplier) do |index|
+            res = Array.new(multiplier) { |index|
               index == node.inputs[1].const_value ? grad : filler
-            end
+            }
             [res]
           end
         when :squeeze
           _reshape_to_input(node, grad)
-        when :expand_dims
-          [_reshape_to_input(node, grad), nil]
         when :concat
           _concat_grad_helper(node, grad, 1, node.inputs.size, 0)
-        when :reshape
-          [ts.reshape(grad, ts.shape(node.inputs[0])), nil]
         when :stack
           res = ts.unstack(grad, num: node.inputs.size, axis: node.options[:axis])
           Array.new(node.inputs.size) { |i| res[i] }
@@ -314,18 +165,8 @@ module TensorStream
           ts.stack(grad, axis: node.options[:axis])
         when :conv2d
           _Conv2DGrad(node, grad)
-        when :cast
-          t = %i[float16 float32 float64]
-          src_type = node.inputs[0].data_type
-          dst_type = grad.data_type
-
-          if t.key?(src_type) && t.key?(dst_type)
-            ts.cast(grad, src_type)
-          end
-
-          nil
         else
-          raise "no derivative op for #{node.operation}"
+          TensorStream::OpMaker.gradient_op(self, node, grad)
         end
       end
     end
@@ -373,8 +214,8 @@ module TensorStream
       zeros = ts.zeros(gradshape, dtype: gdtype)
       xmask = selector_op.call(x, y)
       rx, ry = _broadcast_gradient_args(sx, sy)
-      xgrad = ts.where(xmask, grad, zeros, name: 'x')
-      ygrad = ts.where(xmask, zeros, grad, name: 'y')
+      xgrad = ts.where(xmask, grad, zeros, name: "x")
+      ygrad = ts.where(xmask, zeros, grad, name: "y")
       gx = ts.reshape(ts.reduce_sum(xgrad, rx), sx)
       gy = ts.reshape(ts.reduce_sum(ygrad, ry), sy)
       [gx, gy]
@@ -435,22 +276,22 @@ module TensorStream
 
       shape_0, shape_1 = ts.shape_n([op.inputs[0], op.inputs[1]])
       [
-          _op(:conv2d_backprop_input,
-              shape_0,
-              op.inputs[1],
-              grad,
-              strides: strides,
-              padding: padding,
-              use_cudnn_on_gpu: use_cudnn_on_gpu,
-              data_format: data_format),
-          _op(:conv2d_backprop_filter,
-              op.inputs[0],
-              shape_1,
-              grad,
-              strides: strides,
-              padding: padding,
-              use_cudnn_on_gpu: use_cudnn_on_gpu,
-              data_format: data_format)
+        _op(:conv2d_backprop_input,
+          shape_0,
+          op.inputs[1],
+          grad,
+          strides: strides,
+            padding: padding,
+            use_cudnn_on_gpu: use_cudnn_on_gpu,
+            data_format: data_format),
+        _op(:conv2d_backprop_filter,
+          op.inputs[0],
+          shape_1,
+          grad,
+          strides: strides,
+          padding: padding,
+          use_cudnn_on_gpu: use_cudnn_on_gpu,
+          data_format: data_format),
       ]
     end
   end