tensor_stream 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: adb3c1e45383170db2d5c9f5f3568c251cdddb64
-  data.tar.gz: 99cd0f23353e8f72af2bda3bb84cf4b61c615445
+  metadata.gz: a25008802340137903308c5ec471a1ff4bedfc42
+  data.tar.gz: ba3549b627248f16d1afcea678d844473765d399
 SHA512:
-  metadata.gz: 62eb911e080ba894d6c349870d86286fecbd02f8ad0b762fe33ca311ffbd500f18f78f80520d5e875dc821b1c0f8d556f45fe913142bc02ae33d931af0fb840c
-  data.tar.gz: d599f839bd0d6ba706d3c7fafb3e903a17070af7dfd5dbe733ce21e2641bc4428f8e0d204f649f7f80c843aa48ff2e3c98880c36822a071279de5cd62a39b3dc
+  metadata.gz: 912fc4724db50b4b45a4cf7af11cd28a1df8dc9589a2347fb9c5374ada51cc34c11c179c70fc392ef0bd5c8c6dc41802d8ee4457df47cf7776ee78d92d1f0b74
+  data.tar.gz: 506e86fbd0877c10b51b6106529284c33c6158576e53f2fe720b85213f2ec1577156c614e85a587320e2d8a063dcefee51a59f6325cc1e24a27e8a2e99114ebf

data/README.md

@@ -1,3 +1,5 @@
+[![Gem Version](https://badge.fury.io/rb/tensor_stream.svg)](https://badge.fury.io/rb/tensor_stream)
+
 # TensorStream
 
 A reimplementation of TensorFlow for ruby. This is a ground up implementation with no dependency on TensorFlow. Effort has been made to make the programming style as near to TensorFlow as possible, comes with a pure ruby evaluator by default as well with support for an opencl evaluator.
@@ -94,7 +96,11 @@ tf.session do |sess|
 end
 ```
 
-## python to ruby guide
+You can take a look at spec/tensor_stream/operation_spec.rb for a list of supported ops and various examples and test cases used. Of course these contain only a
+sliver of what TensorFlow can do, so feel free to file a PR to add requested
+ops and test cases.
+
+## Python to Ruby guide
 
 Not all ops are available. Available ops are defined in lib/tensor_stream/ops.rb, corresponding gradients are found at lib/tensor_stream/math_gradients.
 
@@ -122,7 +128,7 @@ w = tf.Variable(0, 'weights')
 Ruby
 
 ```ruby
-w =tf.variable(0, name: 'weights')
+w = tf.variable(0, name: 'weights')
 ```
 
 # Shapes
@@ -150,7 +156,7 @@ W = tf.variable(rand, name: "weight")
 b = tf.variable(rand, name: "bias")
 pred = X * W + b
 cost = tf.reduce_sum(tf.pow(pred - Y, 2)) / ( 2 * 10)
-cost.to_math # "(reduce_sum(|((((Placeholder: * weight) + bias) - Placeholder_2:)^2)|) / 10.0)"
+cost.to_math # "(reduce_sum(|((((Placeholder: * weight) + bias) - Placeholder_2:)^2)|) / 20.0)"
 ```
 
 breakpoints can also be set, block will be evaluated during computation

data/lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb

@@ -0,0 +1,100 @@
+module TensorStream
+  # varoius utility functions for array processing
+  module ArrayOpsHelper
+    def broadcast(input_a, input_b)
+      sa = shape_eval(input_a)
+      sb = shape_eval(input_b)
+
+      return [input_a, input_b] if sa == sb
+
+      # descalar
+      if sa.empty?
+        input_a = [input_a]
+        sa = [1]
+      end
+
+      if sb.empty?
+        input_b = [input_b]
+        sb = [1]
+      end
+
+      target_shape = shape_diff(sa, sb)
+
+      if target_shape
+        input_b = broadcast_dimensions(input_b, target_shape)
+      else
+        target_shape = shape_diff(sb, sa)
+        raise "Incompatible shapes for op #{shape_eval(input_a)} vs #{shape_eval(input_a)}" if target_shape.nil?
+
+        input_a = broadcast_dimensions(input_a, target_shape)
+      end
+
+      [input_a, input_b]
+    end
+
+    # explicit broadcasting helper
+    def broadcast_dimensions(input, dims = [])
+      return input if dims.empty?
+
+      d = dims.shift
+
+      if input.is_a?(Array) && (get_rank(input) - 1) == dims.size
+        row_to_dup = input.collect do |item|
+          broadcast_dimensions(item, dims.dup)
+        end
+
+        row_to_dup + Array.new(d) { row_to_dup }.flatten(1)
+      elsif input.is_a?(Array)
+        Array.new(d) { broadcast_dimensions(input, dims.dup) }
+      else
+        Array.new(d + 1) { input }
+      end
+    end
+
+    # handle 2 tensor math operations
+    def vector_op(vector, vector2, op = ->(a, b) { a + b }, switch = false)
+      if get_rank(vector) < get_rank(vector2) # upgrade rank of A
+        duplicated = Array.new(vector2.size) do
+          vector
+        end
+        return vector_op(duplicated, vector2, op, switch)
+      end
+
+      return op.call(vector, vector2) unless vector.is_a?(Array)
+
+      vector.each_with_index.collect do |item, index|
+        next vector_op(item, vector2, op, switch) if item.is_a?(Array) && get_rank(vector) > get_rank(vector2)
+
+        z = if vector2.is_a?(Array)
+              if index < vector2.size
+                vector2[index]
+              else
+                raise 'incompatible tensor shapes used during op' if vector2.size != 1
+                vector2[0]
+              end
+            else
+              vector2
+            end
+
+        if item.is_a?(Array)
+          vector_op(item, z, op, switch)
+        else
+          switch ? op.call(z, item) : op.call(item, z)
+        end
+      end
+    end
+
+    def shape_diff(shape_a, shape_b)
+      return nil if shape_b.size > shape_a.size
+
+      reversed_a = shape_a.reverse
+      reversed_b = shape_b.reverse
+
+      reversed_a.each_with_index.collect do |s, index|
+        next s if index >= reversed_b.size
+        return nil if reversed_b[index] > s
+        s - reversed_b[index]
+      end.reverse
+    end
+  end
+end

data/lib/tensor_stream/evaluator/ruby_evaluator.rb

@@ -1,4 +1,5 @@
 require 'tensor_stream/evaluator/operation_helpers/random_gaussian'
+require 'tensor_stream/evaluator/operation_helpers/array_ops_helper'
 require 'tensor_stream/math_gradients'
 
 module TensorStream
@@ -25,6 +26,7 @@ module TensorStream
       attr_accessor :retain
 
       include TensorStream::OpHelper
+      include TensorStream::ArrayOpsHelper
 
       def initialize(session, context, thread_pool: nil)
         @session = session
@@ -37,6 +39,8 @@ module TensorStream
         return tensor.map { |t| run(t, execution_context) } if tensor.is_a?(Array)
 
         return tensor if retain.include?(tensor) # if var is in retain don't eval to value
+        
+        tensor = tensor.call() if tensor.is_a?(Proc)
 
         child_context = execution_context.dup
         res = if tensor.is_a?(Operation)
@@ -107,6 +111,11 @@ module TensorStream
           }
 
           call_op(:sign, a, child_context, func)
+        when :logical_and
+          a = complete_eval(a, child_context)
+          b = complete_eval(b, child_context)
+
+          call_vector_op(:greater, a, b, child_context, ->(t, u) { t && u })
         when :equal
           a = complete_eval(a, child_context)
           b = complete_eval(b, child_context)
@@ -188,35 +197,40 @@ module TensorStream
           tensor.items[0].value
         when :reduce_mean
           c = fp_type?(tensor.data_type) ? 0.0 : 0
-          func = lambda { |v|
-            if v.is_a?(Array)
-              v.empty? ? c : (v.reduce(:+) / v.size)
-            else
-              v
+          func = lambda do |arr|
+            return c if arr.nil?
+
+            reduced_val = arr[0]
+            arr[1..arr.size].each do |v|
+              reduced_val = vector_op(reduced_val, v, ->(a, b) { a + b })
             end
-          }
+
+            vector_op(reduced_val, nil, ->(a, _b) { a / arr.size })
+          end
 
           reduction(child_context, tensor, func)
         when :reduce_sum
           c = fp_type?(tensor.data_type) ? 0.0 : 0
-          func = lambda { |v|
-            if v.is_a?(Array)
-              v.empty? ? c : v.reduce(:+)
-            else
-              v
+          func = lambda do |arr|
+            reduced_val = arr[0]
+            arr[1..arr.size].each do |v|
+              reduced_val = vector_op(reduced_val, v, ->(t, u) { t + u })
             end
-          }
+            reduced_val
+          end
 
           reduction(child_context, tensor, func)
         when :reduce_prod
           c = fp_type?(tensor.data_type) ? 1.0 : 1
-          func = lambda { |v|
-            if v.is_a?(Array)
-              v.empty? ? c : v.reduce(:*)
-            else
-              v
+          func = lambda do |arr|
+            return c if arr.nil?
+
+            reduced_val = arr[0]
+            arr[1..arr.size].each do |v|
+              reduced_val = vector_op(reduced_val, v, ->(a, b) { a * b })
             end
-          }
+            reduced_val
+          end
 
           reduction(child_context, tensor, func)
         when :transpose
@@ -316,6 +330,10 @@ module TensorStream
           (Matrix[*matrix_a] * Matrix[*matrix_b]).to_a
         when :gradients
           raise 'not implemented in evaluator' # see TensorStream.gradients instead.
+        when :broadcast_transform
+          a = complete_eval(a, child_context)
+          b = complete_eval(b, child_context)
+          broadcast(a, b)
         when :identity
           complete_eval(a, child_context)
         when :print
@@ -348,6 +366,11 @@ module TensorStream
           b = complete_eval(b, child_context)
 
           call_vector_op(:max, a, b, child_context, ->(t, u) { [t, u].max })
+        when :broadcast_gradient_args
+          a = complete_eval(a, child_context)
+          b = complete_eval(b, child_context)
+
+          get_broadcast_gradient_args(a, b)
         else
           raise "unknown op #{tensor.operation}"
         end.tap do |result|
@@ -362,15 +385,15 @@ module TensorStream
       rescue EvaluatorExcecutionException => e
         raise e
       rescue StandardError => e
-        # a = complete_eval(a, child_context)
-        # b = complete_eval(b, child_context)
-        # puts "name: #{tensor.given_name}"
-        # puts "op: #{tensor.to_math(true, 1)}"
-        # puts "A: #{a}" if a
-        # puts "B: #{b}" if b
-        # binding.pry
+        a = complete_eval(a, child_context)
+        b = complete_eval(b, child_context)
+        puts "name: #{tensor.given_name}"
+        puts "op: #{tensor.to_math(true, 1)}"
+        puts "A: #{a}" if a
+        puts "B: #{b}" if b
+
         puts e.backtrace.join("\n")
-        raise EvaluatorExcecutionException.new(e, tensor), "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math} defined at #{tensor.source}"
+        raise EvaluatorExcecutionException.new(e, tensor), "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true,1)} defined at #{tensor.source}"
       end
 
       def eval_tensor(tensor, child_context)
@@ -425,19 +448,22 @@ module TensorStream
 
       def reduction(child_context, tensor, func)
         val = complete_eval(tensor.items[0], child_context)
-        axis = tensor.options[:axis]
-        keep_dims = tensor.options[:keepdims]
+        axis = complete_eval(tensor.options[:axis], child_context)
+        keep_dims = complete_eval(tensor.options[:keepdims], child_context)
+        rank = get_rank(val)
+        return val if axis && axis.is_a?(Array) && axis.empty?
 
-        res = if axis.is_a?(Array)
-                axis.each do |x|
-                  val = reduce_axis(x, val, keep_dims, child_context, func)
-                end
+        axis = if axis.nil?
+          nil
+        elsif axis.is_a?(Array)
+          return val if axis.empty?
 
-                func.call(val.flatten)
-              else
-                reduce_axis(axis, val, keep_dims, child_context, func)
-              end
-        res
+          axis.map { |a| a < 0 ? rank - a.abs : a }
+        else
+          axis < 0 ? rank - axis.abs : axis
+        end
+
+        reduce_axis(0, axis, val, keep_dims, func)
       end
 
       def arr_pad(arr, paddings, data_type = :float32, rank = 0)
@@ -537,14 +563,41 @@ module TensorStream
           if get_rank(eval_b).zero?
             op.call(eval_a, eval_b)
           else
-            constant_op(eval_b, eval_a, child_context, op, true)
+            vector_op(eval_b, eval_a, op, true)
           end
         elsif get_rank(eval_a) > 0
-          if get_rank(eval_b) > 0
-            vector_op(eval_a, eval_b, child_context, op)
-          else
-            constant_op(eval_a, eval_b, child_context, op)
-          end
+          vector_op(eval_a, eval_b, op)
+        end
+      end
+
+      # determine possible reduction axis to be used
+      def _broadcast_gradient_op(vector_shape1, vector_shape2, level)
+        va_rank = _rank_from_shape(vector_shape1)
+        vb_rank = _rank_from_shape(vector_shape2)
+        return [] if vector_shape1 == vector_shape2 # same shape so no reductions
+
+        shape2_r = vector_shape2.reverse
+
+        vector_shape1.reverse.each_with_index.collect do |s, index|
+          next va_rank - index - 1 if index >= shape2_r.size
+          next nil if shape2_r[index] == s
+          next nil if shape2_r[index] > s
+          va_rank - index - 1
+        end.compact
+      end
+
+      def _rank_from_shape(shape)
+        shape.is_a?(Array) ? shape.size : 0
+      end
+
+      def get_broadcast_gradient_args(input_a, input_b)
+        return [] if get_rank(input_b).zero? && get_rank(input_a).zero?
+        return nil if get_rank(input_b).zero?
+        # ruby scalar
+        if get_rank(input_a).zero?
+          _broadcast_gradient_op(input_b, input_a, 0, true)
+        elsif get_rank(input_a) > 0
+          _broadcast_gradient_op(input_a, input_b, 0)
         end
       end
 
@@ -578,7 +631,7 @@ module TensorStream
       def process_function_op(a, child_context, op)
         # ruby scalar
         if (a.is_a?(Tensor) && a.shape.rank > 0) || a.is_a?(Array)
-          constant_op(a, 0, child_context, op)
+          vector_op(a, 0, op)
         elsif !a.is_a?(Tensor) || a.shape.rank.zero?
           v = run(a, child_context)
           raise FullEvalNotPossible.new, "full eval not possible for #{v.name}" if v.is_a?(Tensor) && !v.is_const
@@ -605,54 +658,29 @@ module TensorStream
         Tensor.cast_dtype(var, placeholder.data_type)
       end
 
-      def reduce_axis(axis, val, keep_dims, child_context, op = ->(v) { v.is_a?(Array) ? v.reduce(:+) : v })
-        val = run(val, child_context)
-        return val.is_a?(Array) ? op.call(val.flatten) : val if axis.nil?
-        return val.transpose.collect { |v| keep_dims ? [op.call(v)] : op.call(v) } if axis.zero?
-        return val.collect { |v| keep_dims ? [op.call(v)] : op.call(v) } if axis == 1
-
-        raise "can't handle with axis > 1 :("
-      end
+      def reduce_axis(current_axis, axis, val, keep_dims, f = ->(a, b) { a + b })
+        return val unless val.is_a?(Array)
 
-      def constant_add(vector, constant)
-        run(vector).collect do |item|
-          if item.is_a?(Array)
-            constant_add(item, constant)
-          elsif item.respond_to?(:value)
-            item.value + constant
-          else
-            item + constant
-          end
+        r = val.collect do |v|
+          reduce_axis(current_axis + 1, axis, v, keep_dims, f)
         end
-      end
 
-      def constant_op(vector, constant, child_context, op = ->(a, b) { a + b }, switch = false)
-        eval_vector = complete_eval(vector, child_context)
-        constant = complete_eval(constant, child_context)
+        should_reduce_axis = axis.nil? || (axis.is_a?(Array) && axis.include?(current_axis)) || (current_axis == axis)
 
-        raise FullEvalNotPossible.new, "full eval not possible for #{eval_vector.name}" if eval_vector.is_a?(Tensor) || constant.is_a?(Tensor)
-
-        eval_vector.each_with_index.collect do |item, index|
-          c = if constant.is_a?(Array)
-              if index < constant.size
-                constant[index]
-              else
-                raise "incompatible tensor shapes used during op" if constant.size != 1
-                constant[0]
-              end
-            else
-              constant
-            end
-          if item.is_a?(Array)
-            constant_op(item, c, child_context, op, switch)
-          elsif item.respond_to?(:value)
-            switch ? op.call(c, item.value) : op.call(item.value, c)
-          else
-            switch ? op.call(c, item) : op.call(item, c)
+        if should_reduce_axis
+          reduced_val = r[0]
+          if r.size > 1
+            reduced_val = f.call(r[0..val.size])
+          elsif r.size == 0
+            reduced_val = f.call(nil)
           end
+          keep_dims ? [ reduced_val ] : reduced_val
+        else
+          r
         end
       end
 
+      # handle 3 tensor math operations
       def call_3way_vector_op(v_a, v_b, v_c, child_context, op = ->(a, b, c) { a + b + c })
         return op.call(v_a, v_b, v_c) unless v_a.is_a?(Array)
 
@@ -667,30 +695,6 @@ module TensorStream
         end
       end
 
-      def vector_op(vector, vector2, child_context, op = ->(a, b) { a + b })
-        v_a = run(vector, child_context)
-        v_b = run(vector2, child_context)
-
-        if get_rank(v_a) < get_rank(v_b) # upgrade rank of A
-          duplicated = Array.new(v_b.size) do
-            v_a
-          end
-          return vector_op(duplicated, v_b, child_context, op)
-        end
-
-        v_a.each_with_index.collect do |item, index|
-          next vector_op(item, v_b, child_context, op) if item.is_a?(Array) && get_rank(v_a) > get_rank(v_b)
-
-          z = index < v_b.size ? v_b[index] : v_b[0]
-
-          if item.is_a?(Array)
-            constant_op(item, z, child_context, op)
-          else
-            item.respond_to?(:value) ? op.call(item.value, z.value) : op.call(item, z)
-          end
-        end
-      end
-
       def all_true?(arr)
         if arr.is_a?(Array)
           arr.each do |a|
@@ -702,21 +706,6 @@ module TensorStream
         !!arr
       end
 
-      def vector_add(vector, vector2, child_context)
-        v_a = run(vector, child_context)
-        v_b = run(vector2, child_context)
-
-        v_a.each_with_index.collect do |item, index|
-          if item.is_a?(Array)
-            constant_add(item, constant)
-          elsif item.respond_to?(:value)
-            item.value + v_b[index].value
-          else
-            item + v_b[index]
-          end
-        end
-      end
-
       def generate_vector(shape, dtype: :float32, generator:)
         if shape.is_a?(Integer)
           Array.new(shape) do

data/lib/tensor_stream/graph.rb

@@ -12,6 +12,11 @@ module TensorStream
     end
 
     def reset
+      @placeholder_counter = 0
+      @const_counter = 0
+      @var_counter = 0
+      @op_counter = 0
+      
       @nodes = {}
       @collections = {
         :"#{GraphKeys::GLOBAL_VARIABLES}" => []
@@ -39,6 +44,7 @@ module TensorStream
       raise 'Placeholder cannot be used when eager_execution is enabled' if @eager_execution && node.is_a?(Placeholder)
       node.name = uniqunify(node.name) if @nodes[node.name]
       @nodes[node.name] = node
+      node.send(:propagate_consumer, node)
       node.value = node.eval if @eager_execution
     end
 

data/lib/tensor_stream/math_gradients.rb

@@ -41,17 +41,31 @@ module TensorStream
         when :log
           (i_cons(1, constant_options_1) / _ds(tensor.items[0])) * grad
         when :tanh
-          (i_cons(1, constant_options_1) - (i_op(:tanh, _ds(tensor.items[0]))**2)) * grad
+          i_op(:mul, (i_cons(1, constant_options_1) - (i_op(:tanh, _ds(tensor.items[0]))**2)), grad, name: 'grad_tanh')
         when :tan
           (i_cons(1, constant_options_1) / (i_op(:cos, _ds(tensor.items[0]))**2)) * grad
         when :sin
-          i_op(:cos, tensor.items[0]) * grad
+          i_op(:mul, i_op(:cos, tensor.items[0]), grad, name: 'grad_sin')
         when :sqrt
           i_cons(1, constant_options_1) / (i_cons(2, constant_options_1) * i_op(:sqrt, _ds(tensor.items[0]))) * grad
         when :cos
           -i_op(:sin, tensor.items[0]) * grad
         when :add
-          _grad_with_broadcast(tensor, wrt_dx, ->(a, b) { i_op(:add, a, b, name: 'grad_sum') }, options)
+          # rx = op(:shape, tensor.items[0])
+          # ry = op(:shape, tensor.items[1])
+
+          # ones_a = op(:ones_like, tensor.items[0])
+          # ones_b = op(:ones_like, tensor.items[1])
+          # inputs = _broadcast_transform(grad * ones_a, grad2 * ones_b)
+          # sx, sy = _broadcast_gradient_args(rx, ry)
+
+          # keep_dims_x = op(:rank, inputs[0]) == op(:rank, tensor.items[0])
+          # keep_dims_y = op(:rank, inputs[1]) == op(:rank, tensor.items[1])
+
+          # add_x = op(:reduce_sum, inputs[0], nil, axis: sy, keepdims: keep_dims_x)
+          # add_y = op(:reduce_sum, inputs[1], nil, axis: sx, keepdims: keep_dims_y)
+          # _filtered_sum(add_x, add_y, wrt_dx)
+          _grad_with_broadcast(tensor, wrt_dx, ->(a, b) { i_op(:add, a, b, name: 'grad_add') }, options)
         when :sub
           _grad_with_broadcast(tensor, wrt_dx, ->(a, b) { i_op(:sub, a, b, name: 'grad_sub') }, options)
         when :pow
@@ -69,7 +83,15 @@ module TensorStream
           _reduce_when_necessary(gx + gy, wrt_dx)
         when :mul
           # apply the product rule
-          _reduce_when_necessary(grad * _ds(tensor.items[1]) + _ds(tensor.items[0]) * grad2, wrt_dx)
+          rx = op(:shape, tensor.items[0])
+          ry = op(:shape, tensor.items[1])
+          sx, sy = _broadcast_gradient_args(rx, ry)
+          inputs = _broadcast_transform(tensor.items[0], tensor.items[1])
+          keep_dims_x = op(:rank, inputs[0]) == op(:rank, tensor.items[0])
+          keep_dims_y = op(:rank, inputs[1]) == op(:rank, tensor.items[1])
+
+          _filtered_sum(op(:reduce_sum, grad * _ds(inputs[1]), nil, axis: sy, keepdims: keep_dims_x),
+                        op(:reduce_sum, _ds(inputs[0]) * grad2, nil, axis: sx, keepdims: keep_dims_y), wrt_dx)
         when :reduce_mean
           input_size = i_op(:reduce_prod, i_op(:shape, tensor.items[0]))
           output_size = i_op(:reduce_prod, i_op(:shape, tensor))
@@ -96,9 +118,6 @@ module TensorStream
           matmul_db = i_op(:matmul, tensor.items[0], identity_1, transpose_a: true,
                                                                  pad_zeros: true,
                                                                  name:        'matrix_dy')
-
-          zero_vect = i_op(:zeros_like, wrt_dx, nil, name: 'zero_vect')
-
           # matmul_db = op(:transpose, matmul_db, nil).first
 
           # begin_a = op(:zeros, op(:rank, matmul_db), nil, data_type: :int32, name: 'begin_a')
@@ -113,9 +132,7 @@ module TensorStream
 
           # norm_a = i_op(:cond, norm_a[0], norm_a, pred: i_op(:rank, matmul_da) > i_op(:rank, derivative_a))
           # norm_b = i_op(:cond, norm_b[0], norm_b, pred: i_op(:rank, matmul_db) > i_op(:rank, derivative_b))
-
-          i_op(:cond, norm_a, zero_vect, pred: i_op(:reduce_sum, norm_a) != 0) + i_op(:cond, norm_b, zero_vect, pred: i_op(:reduce_sum, norm_b) != 0)
-          #  m.breakpoint! { |t, a, b, v|  binding.pry }
+          _filtered_sum(norm_a, norm_b, wrt_dx)
         else
           raise "no derivative implementation found for op #{tensor.operation}"
         end
@@ -161,5 +178,19 @@ module TensorStream
       reduced = op(:reduce_sum, tensor, nil, axis: 0)
       op(:cond, ->{ reduced }, tensor, pred: rank > dx_rank)
     end
+
+    def self._broadcast_gradient_args(input_a, input_b)
+      [op(:broadcast_gradient_args, input_a, input_b), op(:broadcast_gradient_args, input_b, input_a)]
+    end
+
+    def self._broadcast_transform(input_a, input_b)
+      op(:broadcast_transform, input_a, input_b)
+    end
+
+    # filter out zero arrays
+    def self._filtered_sum(input_a, input_b, wrt_dx)
+      zero_vect = op(:zeros_like, wrt_dx)
+      (i_op(:cond, input_a, zero_vect, pred: i_op(:reduce_sum, input_a) != 0) + i_op(:cond, input_b, zero_vect, pred: i_op(:reduce_sum, input_b) != 0))
+    end
   end
 end

data/lib/tensor_stream/operation.rb

@@ -2,6 +2,7 @@ module TensorStream
   # TensorStream class that defines an operation
   class Operation < Tensor
     attr_accessor :name, :operation, :items, :rank, :options
+    attr_reader :outputs
 
     def initialize(operation, input_a, input_b, options = {})
       @graph = options[:graph] || TensorStream.get_default_graph
@@ -51,7 +52,7 @@ module TensorStream
 
     def set_data_type(passed_data_type)
       case operation
-      when :greater, :less, :equal
+      when :greater, :less, :equal, :not_equal, :greater_equal, :less_equal
         :boolean
       when :shape, :rank
         :int32
@@ -150,6 +151,8 @@ module TensorStream
         "#{sub_item} == #{auto_math(items[1], name_only, max_depth - 1)}"
       when :not_equal
         "#{sub_item} != #{auto_math(items[1], name_only, max_depth - 1)}"
+      when :logical_and
+        "#{sub_item} && #{auto_math(items[1], name_only, max_depth - 1)}"
       when :sqrt
         "sqrt(#{sub_item})"
       when :zeros_like
@@ -161,7 +164,7 @@ module TensorStream
       when :cast
         "cast(#{auto_math(sub_item)}, #{data_type})"
       else
-        raise "math form for #{operation}"
+        raise "no math form for #{operation} defined"
       end
     end
 
@@ -171,6 +174,14 @@ module TensorStream
 
     private
 
+    def propagate_consumer(consumer)
+      super(consumer)
+
+      @items.compact.each do |item|
+        item.send(:propagate_consumer, consumer) if item.name!=self.name
+      end
+    end
+
     def set_name
       "#{@operation}#{graph.get_operation_counter}:#{@rank}"
     end

data/lib/tensor_stream/ops.rb

@@ -49,7 +49,7 @@ module TensorStream
     end
 
     def eye(num_rows, num_columns: nil, dtype: :float32, name: nil)
-      op(:eye, num_rows, num_columns || num_rows, data_type: dtype, name: name, preserve_params_type: true)
+      op(:eye, num_rows, num_columns || num_rows, data_type: dtype, name: name)
     end
 
     def shape(input, name: nil, out_type: :int32)
@@ -80,6 +80,10 @@ module TensorStream
       op(:less, input_a, input_b, name: name)
     end
 
+    def logical_and(input_a, input_b, name: nil)
+      op(:logical_and, input_a, input_b, name: name)
+    end
+
     def greater(input_a, input_b, name: nil)
       op(:greater, input_a, input_b, name: name)
     end

data/lib/tensor_stream/tensor.rb

@@ -5,7 +5,9 @@ module TensorStream
   class Tensor
     include OpHelper
 
-    attr_accessor :name, :data_type, :shape, :rank, :native_buffer, :is_const, :value, :breakpoint, :internal, :source, :given_name, :graph
+    attr_accessor :name, :data_type, :shape, :rank, :native_buffer, :is_const,
+                  :value, :breakpoint, :internal, :source, :given_name, :graph,
+                  :consumers
 
     def initialize(data_type, rank, shape, options = {})
       @data_type = data_type
@@ -104,6 +106,14 @@ module TensorStream
       op(:less_equal, self, other)
     end
 
+    def and(other)
+      op(:logical_and, self, other)
+    end
+
+    def matmul(other)
+      op(:matmul, self, other)
+    end
+
     def collect(&block)
       @value.collect(&block)
     end
@@ -204,12 +214,24 @@ module TensorStream
     end
 
     def breakpoint!(&block)
-      @breakpoint = block
       self
     end
 
+    def print!(message)
+      op(:print, self, self, message: message)
+    end
+
     protected
 
+    def add_consumer(consumer)
+      @consumers ||= []
+      @consumers << consumer.name if !@consumers.include?(consumer.name) && consumer.name!=self.name
+    end
+
+    def propagate_consumer(consumer)
+      add_consumer(consumer)
+    end
+
     def format_source(trace)
       trace.reject { |c| c.to_s.include?(File.join('lib', 'tensor_stream')) }.first
     end

data/lib/tensor_stream/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
-  VERSION = '0.1.3'.freeze
+  VERSION = '0.1.4'.freeze
 
   def self.version
     VERSION

data/samples/test.py

@@ -0,0 +1,44 @@
+import tensorflow as tf
+
+test_inputs = [
+  [0.5937, 0.2343, 1.4332, 0.4395],
+  [-1.0227, -0.6915, 1.2367, 0.3452],
+  [-0.5675, 1.0374, 1.0429, 0.8839],
+  [-0.1066, -0.0469, -1.6317, -1.4836],
+  [0.7835, -3.0105, 1.713, -0.4536],
+  [-0.3076, 1.3662, -0.6537, 0.0905],
+  [-0.2459, 0.2243, -2.7048, 0.848],
+]
+
+num_inputs = 4
+num_neurons = 5
+inputs = tf.placeholder("float", shape=(None, num_inputs))
+biases = tf.constant([0.5012, 1.302, -1.6217, 0.669, 0.1494], name='b1')
+biases2 = tf.constant([0.2012, 1.102, -1.5217, 0.469, 0.0494], name='b2')
+
+weights = tf.constant([
+  [-0.9135, 1.0376, 0.8537, 0.4376, 1.3255],
+  [-0.5921, -1.4081, 1.0614, -0.5283, 1.1832],
+  [0.7285, -0.7844, 0.1793, -0.5275, -0.4426],
+  [-1.4976, 0.4433, 2.2317, -2.0479, 0.7791]], name='w')
+
+weights_layer2 = tf.constant([
+  [-1.0465, -0.8766, 1.6849, -0.6625, 0.7928],
+  [2.0412, 1.3564, 0.7905, 0.6434, -2.5495],
+  [2.4276, -0.6893, -1.5917, 0.0911, 0.9112],
+  [-0.012, 0.0794, 1.3829, -1.018, -0.9328],
+  [0.061, 0.9791, -2.1727, -0.9553, -1.434]], name='w2')
+
+
+sess = tf.Session()
+
+layer_1 =  tf.matmul(inputs, weights) + biases
+neural_net = tf.matmul(layer_1, weights_layer2) + biases2
+
+output = sess.run(neural_net, feed_dict={ inputs: test_inputs })
+
+g = tf.gradients(neural_net, [weights, biases])
+g2 = tf.gradients(neural_net, [weights_layer2, biases2])
+
+weight_gradient, biases_gradient = sess.run(g, feed_dict = { inputs: test_inputs })
+weight_gradient2, biases_gradient2 = sess.run(g2, feed_dict: { inputs => test_inputs })

data/tensor_stream.gemspec

@@ -36,6 +36,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "awesome_print"
   spec.add_development_dependency "rubocop"
   spec.add_development_dependency "pry-byebug"
+  spec.add_development_dependency "byepry"
+  spec.add_development_dependency "colorize"
   spec.add_dependency "deep_merge"
   spec.add_dependency "concurrent-ruby"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tensor_stream
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Joseph Emmanuel Dayo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-05-12 00:00:00.000000000 Z
+date: 2018-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: byepry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: deep_merge
   requirement: !ruby/object:Gem::Requirement
@@ -147,6 +175,7 @@ files:
 - lib/tensor_stream.rb
 - lib/tensor_stream/control_flow.rb
 - lib/tensor_stream/evaluator/evaluator.rb
+- lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb
 - lib/tensor_stream/evaluator/operation_helpers/random_gaussian.rb
 - lib/tensor_stream/evaluator/ruby_evaluator.rb
 - lib/tensor_stream/graph.rb
@@ -171,6 +200,7 @@ files:
 - samples/iris.rb
 - samples/linear_regression.rb
 - samples/raw_neural_net_sample.rb
+- samples/test.py
 - tensor_stream.gemspec
 homepage: http://www.github.com/jedld/tensor_stream
 licenses:
@@ -193,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: A Pure ruby tensorflow implementation