tensor_stream 0.5.1 → 0.6.0

data/lib/tensor_stream/math_gradients.rb

@@ -16,22 +16,17 @@ module TensorStream
         node.consumers.include?(tensor.name) || node.equal?(tensor)
       end.compact + [wrt_dx.name]
 
-      grad = i_op(:ones_like, wrt_dx)
+      grad = i_op(:fill, tf.shape(tensor), tf.constant(1, dtype: wrt_dx.data_type))
 
-      result = _propagate(grad, tensor, wrt_dx, nodes_to_compute, options[:stop_gradients] || [])
-      i_op(:truncate, result, tf.shape(wrt_dx))
+      _propagate(grad, tensor, wrt_dx, nodes_to_compute, options[:stop_gradients] || [] ) || i_op(:zeros_like, wrt_dx)
     end
 
     def self._propagate(grad, tensor, stop_tensor, nodes_to_compute, stop_gradients = [])
-      return grad * i_op(:ones_like, stop_tensor) if stop_tensor.equal?(tensor)
-      return i_op(:zeros_like, stop_tensor) if stop_gradients && _include?(stop_gradients, tensor)
-      return i_op(:zeros_like, stop_tensor) unless tensor.is_a?(Operation)
+      return grad if stop_tensor.equal?(tensor)
+      return nil if stop_gradients && _include?(stop_gradients, tensor)
+      return nil unless tensor.is_a?(Operation)
 
-      computed_op = if _op_supports_broadcast?(tensor)
-                      _compute_derivative(tensor, _broadcast_transform(tensor, grad)[1])
-                    else
-                      _compute_derivative(tensor, grad)
-                    end
+      computed_op = _compute_derivative(tensor, grad)
 
       if computed_op.is_a?(Array)
         partials = []
@@ -43,9 +38,9 @@ module TensorStream
           end
         end
 
-        partials.reduce(:+)
+        partials.compact.reduce(:+)
       else
-        return tf.zeros_like(stop_tensor) if computed_op.nil?
+        return nil if computed_op.nil?
         _propagate(computed_op, tensor.inputs[0], stop_tensor, nodes_to_compute, stop_gradients)
       end
     end
@@ -57,66 +52,74 @@ module TensorStream
 
         case node.operation
         when :add
-          return [grad, grad] if _shapes_fully_specified_and_equal(x, y)
-
+          return [grad, grad] if shapes_fully_specified_and_equal(x, y)
           sx = tf.shape(x, name: 'add/shape_x')
           sy = tf.shape(y, name: 'add/shape_y')
           rx, ry = _broadcast_gradient_args(sx, sy)
-          keep_dims_x = tf.rank(x) == tf.rank(grad)
-          keep_dims_y = tf.rank(y) == tf.rank(grad)
 
-          [tf.reduce_sum(grad, rx, name: 'add/reduce_sum_x', keepdims: keep_dims_x),
-          tf.reduce_sum(grad, ry, name: 'add/reduce_sum_y', keepdims: keep_dims_y)]
+          [ tf.reshape(tf.reduce_sum(grad, rx, name: 'add/reduce_sum_x'),sx),
+            tf.reshape(tf.reduce_sum(grad, ry, name: 'add/reduce_sum_y'),sy) ]
         when :sub
-          return [grad, -grad] if _shapes_fully_specified_and_equal(x, y)
+          return [grad, -grad] if shapes_fully_specified_and_equal(x, y)
 
           sx = tf.shape(x, name: 'sub/shape_x')
           sy = tf.shape(y, name: 'sub/shape_y')
           rx, ry = _broadcast_gradient_args(sx, sy)
-          [tf.reduce_sum(grad, rx), -tf.reduce_sum(grad, ry)]
+
+          [ tf.reshape(tf.reduce_sum(grad, rx, name: 'add/reduce_sub_x'),sx),
+            -tf.reshape(tf.reduce_sum(grad, ry, name: 'add/reduce_sub_y'),sy) ]
         when :mul
           sx = tf.shape(x)
           sy = tf.shape(y)
           rx, ry = _broadcast_gradient_args(sx, sy)
 
-          [ tf.reduce_sum(tf.mul(grad, y), rx),
-            tf.reduce_sum(tf.mul(x, grad), ry)]
+          [ tf.reshape(tf.reduce_sum(tf.mul(grad, y), rx), sx),
+            tf.reshape(tf.reduce_sum(tf.mul(x, grad), ry), sy)]
         when :div
           sx = i_op(:shape, x)
           sy = i_op(:shape, y)
           rx, ry = _broadcast_gradient_args(sx, sy)
 
-          [tf.reduce_sum(tf.div(grad, y), rx),
-          tf.reduce_sum(grad * tf.div(tf.div(-x, y), y),
-                                  ry)]
+          [
+            tf.reshape(tf.reduce_sum(tf.div(grad, y), rx), sx),
+            tf.reshape(tf.reduce_sum(grad * tf.div(tf.div(-x, y), y),
+                                  ry), sy)]
+        when :mod
+          sx = tf.shape(x)
+          sy = tf.shape(y)
+          rx, ry = _broadcast_gradient_args(sx, sy)
+          floor_xy = tf.floor_div(x, y)
+          gx = tf.reshape(tf.reduce_sum(grad, rx), sx)
+          gy = tf.reshape(tf.reduce_sum(grad * tf.negative(floor_xy), ry), sy)
+
+          [gx, gy]
+        when :squared_difference
+          sx = i_op(:shape, x)
+          sy = i_op(:shape, y)
+          rx, ry = _broadcast_gradient_args(sx, sy)
+
+          x_grad = tf.mul(2.0, grad) * (x - y)
+
+          [ tf.reshape(tf.reduce_sum(x_grad, rx), sx),
+            tf.reshape(-tf.reduce_sum(x_grad, ry), sy)]
         when :matmul
           t_a = node.options[:transpose_a]
           t_b = node.options[:transpose_b]
 
-          s0 =  tf.shape(x)
-          s1 =  tf.shape(y)
-
-          identity_0 = tf.ones([ s0[0], s1[1] ], dtype: x.data_type, name: 'matmul/identity0')
-          identity_1 = tf.ones([ s0[0], s1[1] ], dtype: y.data_type, name: 'matmul/identity1')
-
-          grad_a, grad_b = nil
           if !t_a && !t_b
-            grad_a = tf.matmul(identity_0, y, transpose_b: true)
-            grad_b = tf.matmul(x, identity_1, transpose_a: true)
+            grad_a = tf.matmul(grad, y, transpose_b: true)
+            grad_b = tf.matmul(x, grad, transpose_a: true)
           elsif !ta && tb
-            grad_a = tf.matmul(identity_0, y)
-            grad_b = tf.matmul(identity_1, x, transpose_a: true)
+            grad_a = tf.matmul(grad, y)
+            grad_b = tf.matmul(grad, x, transpose_a: true)
           elsif t_a && !t_b
-            grad_a = tf.matmul(y, identity_0, transpose_b: true)
-            grad_b = tf.matmul(x, identity_1)
+            grad_a = tf.matmul(y, grad, transpose_b: true)
+            grad_b = tf.matmul(x, grad)
           elsif t_a && t_b
-            grad_a = tf.matmul(y, identity_0, transpose_a: true, transpose_b: true)
-            grad_b = tf.matmul(identity_1, x, transpose_a: true, transpose_b: true)
+            grad_a = tf.matmul(y, grad, transpose_a: true, transpose_b: true)
+            grad_b = tf.matmul(grad, x, transpose_a: true, transpose_b: true)
           end
 
-          grad_a = i_op(:mul, grad, grad_a, name: 'matmul/grad_a_norm_mul_da')
-          grad_b = i_op(:mul, grad, grad_b, name: 'matmul/grad_b_norm_mul_db')
-
           [grad_a, grad_b]
         when :sin
           grad * tf.cos(x)
@@ -153,7 +156,7 @@ module TensorStream
           -grad
         when :exp
           grad * node
-        when :identity
+        when :identity, :print
           grad
         when :sum
           _sum_grad(x, y, grad)
@@ -175,7 +178,7 @@ module TensorStream
           y_cond = i_op(:cond, i_op(:zeros_like, x), i_op(:ones_like, x), pred: node.options[:pred])
           [x_cond * grad, y_cond * grad]
         when :mean
-          sum_grad = _sum_grad(x, y, grad)
+          sum_grad  = _sum_grad(x, y, grad)[0]
           input_shape = tf.shape(x)
           output_shape = tf.shape(node)
           factor = _safe_shape_div(tf.reduce_prod(input_shape), tf.reduce_prod(output_shape))
@@ -186,6 +189,10 @@ module TensorStream
           i_op(:sigmoid_grad, x, grad)
         when :softmax
           i_op(:softmax_grad, x, grad)
+        when :softmax_cross_entropy_with_logits_v2
+          # -grad * tf.reciprocal(i_op(:softmax, x))
+          [i_op(:softmax_cross_entropy_with_logits_v2_grad, x, y, grad), nil]
+          # i_op(:softmax_grad, x, -grad * tf.reciprocal(i_op(:softmax, x)))
         when :floor, :ceil
           # non differentiable
           nil
@@ -202,7 +209,8 @@ module TensorStream
     end
 
     def self._broadcast_gradient_args(input_a, input_b)
-      [_op(:broadcast_gradient_args, input_b, input_a), _op(:broadcast_gradient_args, input_a, input_b)]
+      res = _op(:broadcast_gradient_args, input_a, input_b)
+      [res[0], res[1]]
     end
 
     def self._broadcast_transform(input_a, input_b)
@@ -210,11 +218,18 @@ module TensorStream
     end
 
     def self._safe_shape_div(x, y)
-      x / tf.maximum(y, 1)
+      _op(:floor_div, x , tf.maximum(y, 1))
     end
 
     def self._sum_grad(x, y, grad)
-      tf.ones_like(grad) * grad
+      input_shape = _op(:shape, x)
+      output_shape_kept_dims = tf.reduced_shape(input_shape, y)
+      tile_scaling = _safe_shape_div(input_shape, output_shape_kept_dims)
+      new_grad = _op(:reshape, grad, output_shape_kept_dims)
+
+      grad = _op(:cond, _op(:fill, input_shape, grad) , _op(:tile, new_grad, tile_scaling), pred: _op(:rank, grad) == 0 )
+
+      [grad, nil ]
     end
 
     def self._op_supports_broadcast?(node)
@@ -233,20 +248,5 @@ module TensorStream
       arr.each { |a| return true if a.equal?(obj) }
       false
     end
-
-    def self._shapes_fully_specified_and_equal(x, y)
-     return false if !_shape_full_specified(x) || !_shape_full_specified(y)
-     return false if x.shape.shape != y.shape.shape
-     
-     true
-    end
-
-    def self._shape_full_specified(tensor)
-      return false if tensor.shape.nil?
-      return false if tensor.shape.shape.nil?
-
-      tensor.shape.shape.each { |s| return false if s.nil? }
-      true
-    end
   end
 end

data/lib/tensor_stream/nn/nn_ops.rb

@@ -15,12 +15,16 @@ module TensorStream
     end
 
     def self.softmax_cross_entropy_with_logits(labels: nil, logits: nil, name: nil)
-      TensorStream.name_scope(name, default: 'softmax_cross_entropy_with_logits', values: [logits, labels]) do |name|
+      softmax_cross_entropy_with_logits_v2(labels, logits, name)
+    end
+
+    def self.softmax_cross_entropy_with_logits_v2(labels: nil, logits: nil, name: nil)
+      TensorStream.name_scope(name, default: 'softmax_cross_entropy_with_logits', values: [logits, labels]) do
         tf = TensorStream
         logits = tf.convert_to_tensor(logits, name: 'logits')
         labels = tf.convert_to_tensor(labels, name: 'labels')
         labels = tf.cast(labels, logits.dtype)
-        softmax_logits = -tf.log(softmax(logits)) * labels
+        softmax_logits = _op(:softmax_cross_entropy_with_logits_v2, logits, labels)
         tf.reduce_sum(softmax_logits, tf.rank(logits) - 1)
       end
     end

data/lib/tensor_stream/operation.rb

@@ -69,10 +69,12 @@ module TensorStream
 
     def set_data_type(passed_data_type)
       case operation
+      when :fill
+        @inputs[1].data_type
       when :greater, :less, :equal, :not_equal, :greater_equal, :less_equal, :logical_and
         :boolean
       when :shape, :rank
-        :int32
+        options[:out_type] || :int32
       when :random_normal, :random_uniform, :glorot_uniform
         passed_data_type || :float32
       when :index
@@ -282,13 +284,25 @@ module TensorStream
     def propagate_consumer(consumer)
       super
       @inputs.compact.each do |input|
-        input.send(:propagate_consumer, consumer) if input.name != name
+        if input.is_a?(Array)
+          input.flatten.compact.each do |t|
+            t.send(:propagate_consumer, consumer) if t.is_a?(Tensor)
+          end
+        else
+          input.send(:propagate_consumer, consumer) if input.name != name
+        end
       end
     end
 
     def propagate_outputs
       @inputs.compact.each do |input|
-        input.send(:setup_output, self) if input.name != self.name
+        if input.is_a?(Array)
+          input.flatten.compact.each do |t|
+            t.send(:setup_output, self) if t.is_a?(Tensor)
+          end
+        else
+          input.send(:setup_output, self) if input.is_a?(Tensor) && (input.name != self.name)
+        end
       end
     end
 

data/lib/tensor_stream/ops.rb

@@ -89,6 +89,9 @@ module TensorStream
     ##
     # This operation returns a 1-D integer tensor representing the shape of input
     def shape(input, name: nil, out_type: :int32)
+      return constant(shape_eval(input, out_type), dtype: out_type, name: name) if input.is_a?(Array)
+      return constant(input.shape.shape, dtype: out_type, name: "Shape/#{input.name}") if shape_full_specified(input)
+
       _op(:shape, input, nil, name: name, out_type: out_type)
     end
 
@@ -287,6 +290,24 @@ module TensorStream
       _op(:sub, input_a, input_b, name: name)
     end
 
+    ##
+    # Returns element-wise remainder of division.
+    def mod(input_a, input_b, name: nil)
+      input_a, input_b = check_data_types(input_a, input_b)
+      _op(:mod, input_a, input_b, name: name)
+    end
+
+    ##
+    # Returns element-wise integer divistion.
+    def floor_div(input_a, input_b, name: nil)
+      input_a, input_b = check_data_types(input_a, input_b)
+      _op(:floor_div, input_a, input_b, name: name)
+    end
+
+    def range(start, limit, delta = 1, dtype: nil, name: 'range')
+      _op(:range, start, limit, delta, data_type: dtype, name: name)
+    end
+
     ##
     # Returns x - y element-wise.
     #
@@ -476,6 +497,19 @@ module TensorStream
       _op(:exp, input, nil, name: name)
     end
 
+    ##
+    # Creates a tensor filled with a scalar value.
+    #
+    # This operation creates a tensor of shape dims and fills it with value.
+    #
+    # For example:
+    # Output tensor has shape [2, 3].
+    # fill([2, 3], 9) => [[9, 9, 9]
+     #                    [9, 9, 9]]
+    def fill(dims, value, name: nil)
+      _op(:fill, dims, value, name: name)
+    end
+
     ##
     # Computes sigmoid of x element-wise.
     def sigmoid(input, name: nil)
@@ -512,5 +546,18 @@ module TensorStream
     def check_numerics(tensor, message, name: nil)
       _op(:check_numerics, tensor, nil, message: message, name: name)
     end
+
+    def size(tensor, name: nil, out_type: :int32)
+      _op(:size, tensor, name: name, out_type: out_type)
+    end
+
+    def squared_difference(input_a, input_b, name: nil)
+      _op(:squared_difference, input_a, input_b, name: name)
+    end
+
+    def broadcast_gradient_args(shape_a, shape_b, name: nil)
+      op_result = _op(:broadcast_gradient_args, shape_a, shape_b, name: name)
+      [op_result[0], op_result[1]]
+    end
   end
 end

data/lib/tensor_stream/session.rb

@@ -71,7 +71,7 @@ module TensorStream
       end
       result = args.collect do |e|
         value = delegate_to_evaluator(e, context, {})
-        value.respond_to?(:to_ruby) ? value.to_ruby : value
+        recursive_eval(value)
       end
       result.size == 1 ? result.first : result
     end
@@ -122,6 +122,14 @@ module TensorStream
 
     protected
 
+    def recursive_eval(value, depth = 2)
+      if value.is_a?(Array) && depth > 0
+        value.collect { |v| recursive_eval(v, depth - 1) }
+      else
+        value.respond_to?(:to_ruby) ? value.to_ruby : value
+      end
+    end
+
     def assign_evaluator(tensor)
       device = @evaluator_classes.map do |klass|
         next nil if tensor.is_a?(Operation) && !klass.ops.include?(tensor.operation.to_sym)

data/lib/tensor_stream/tensor.rb

@@ -87,6 +87,19 @@ module TensorStream
       _op(:negate, self, nil)
     end
 
+    def %(other)
+      _a, other = TensorStream.check_data_types(self, other)
+      _op(:mod, self, TensorStream.convert_to_tensor(other, dtype: data_type))
+    end
+
+    def floor
+      TensorStream.floor(self)
+    end
+
+    def ceil
+      TensorStream.ceil(self)
+    end
+
     def ==(other)
       _a, other = TensorStream.check_data_types(self, other)
       _op(:equal, self, other)
@@ -199,6 +212,8 @@ module TensorStream
         :int32
       elsif value.is_a?(Array)
         return detect_type(value[0])
+      elsif value.is_a?(Tensor)
+        value.data_type
       else
         :float32
       end

data/lib/tensor_stream/utils.rb

@@ -141,6 +141,10 @@ module TensorStream
       TensorStream::ControlFlow.new(:group, inputs, nil, name: name)
     end
 
+    def dynamic_stitch(indices, data, name: nil)
+      TensorStream::DynamicStitch.new(:dynamic_stitch, [indices, data], name: name)
+    end
+
     def get_variable(name, dtype: nil, shape: nil, initializer: nil, trainable: true, collections: nil)
       TensorStream::Variable.new(dtype || :float32, nil, shape, collections: collections, name: name, initializer: initializer, trainable: trainable)
     end
@@ -204,7 +208,7 @@ module TensorStream
         input_a = convert_to_tensor(input_a)
         input_b = convert_to_tensor(input_b)
       end
-
+      
       if norm_dtype(input_a.data_type) != norm_dtype(input_b.data_type)
         raise "Value Error: Tensor conversion requested dtype #{input_a.data_type} for tensor type #{input_b.data_type}" 
       end

data/lib/tensor_stream/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
-  VERSION = '0.5.1'.freeze
+  VERSION = '0.6.0'.freeze
 
   def self.version
     VERSION

data/lib/tensor_stream.rb

@@ -16,6 +16,7 @@ require 'tensor_stream/variable'
 require 'tensor_stream/operation'
 require 'tensor_stream/placeholder'
 require 'tensor_stream/control_flow'
+require 'tensor_stream/dynamic_stitch'
 require 'tensor_stream/trainer'
 require 'tensor_stream/nn/nn_ops'
 require 'tensor_stream/evaluator/evaluator'