tensor_stream 0.5.1 → 0.6.0

data/lib/tensor_stream/evaluator/opencl/opencl_template_helper.rb

@@ -86,6 +86,8 @@ class OpenclTemplateHelper
       '-'
     when 'mul'
       '*'
+    when 'mod'
+      '%'
     else
       raise "unsupported op #{op}"
     end

data/lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb

@@ -23,7 +23,7 @@ module TensorStream
       slice_tensor(input, start, target_shape)
     end
 
-    def reduced_shape(input_shape, axes)
+    def _reduced_shape(input_shape, axes)
       return [] if axes.nil? # reduce to scalar
       axes = [ axes ] unless axes.is_a?(Array)
       return input_shape if axes.empty?
@@ -166,25 +166,35 @@ module TensorStream
       get_rank(value[0], rank + 1)
     end
 
+    def last_axis(arr)
+      all_items = []
+      if get_rank(arr) <=2
+        return arr
+      else
+        arr.each do |sub|
+          all_items += last_axis(sub)
+        end
+      end
+      all_items
+    end
+
     def softmax(arr)
       return arr if arr.empty?
 
-      sum = if !arr[0].is_a?(Array)
-        arr.map { |a| Math.exp(a - arr.max) }.reduce(:+)
-      end
-
-      arr.collect do |input|
-        if input.is_a?(Array)
-          softmax(input)
-        else
-          Math.exp(input - arr.max) / sum
+      if !arr[0].is_a?(Array)
+        c = arr.max
+        arr = arr.map { |a| Math.exp(a - c) }
+        sum = arr.reduce(:+)
+        arr.collect do |input|
+          input / sum
         end
+      else
+        arr.collect { |input| softmax(input) }
       end
     end
 
     def softmax_grad(arr)
       return arr if arr.empty?
-
       arr.each_with_index.collect do |input, index|
         if input.is_a?(Array)
           softmax_grad(input)

data/lib/tensor_stream/evaluator/ruby_evaluator.rb

@@ -160,6 +160,28 @@ module TensorStream
         slice_tensor(input, start, size)
       end
 
+      def merge_dynamic_stitch(merged, indexes, data)
+        indexes.each_with_index do |ind, m|
+          if ind.is_a?(Array)
+            merge_dynamic_stitch(merged, ind, data[m])
+          else
+            merged[ind] = data[m]
+          end
+        end
+      end
+
+      register_op :flow_dynamic_stitch, noop: true do |context, tensor, inputs|
+        indexes, data = inputs
+        merged = []
+        merge_dynamic_stitch(merged, indexes, data)
+        merged
+      end
+
+      register_op :size do |_context, tensor, inputs|
+        input = inputs[0]
+        Tensor.cast_dtype(input.flatten.size, tensor.options[:out_type])
+      end
+
       register_op :negate, no_eval: true do |context, _tensor, inputs|
         call_vector_op(:negate, inputs[0], nil, context, ->(t, _u) { -t })
       end
@@ -174,6 +196,20 @@ module TensorStream
         call_vector_op(:sub, a, b, context, ->(t, u) { t - u })
       end
 
+      register_op :mod, no_eval: true do |context, _tensor, inputs|
+        a, b = inputs
+        call_vector_op(:sub, a, b, context, ->(t, u) { t % u })
+      end
+
+      register_op :floor_div, no_eval: true do |context, tensor, inputs|
+        a, b = inputs
+        if fp_type?(tensor.data_type)
+          call_vector_op(:sub, a, b, context, ->(t, u) { (t / u).to_i.to_f })
+        else
+          call_vector_op(:sub, a, b, context, ->(t, u) { t / u })
+        end
+      end 
+
       register_op :mul, no_eval: true do |context, _tensor, inputs|
         a, b = inputs
         call_vector_op(:mul, a, b, context, ->(t, u) { t * u })
@@ -184,6 +220,11 @@ module TensorStream
         call_vector_op(:pow, a, b, context, ->(t, u) { t**u })
       end
 
+      register_op :squared_difference, no_eval: true do |context, _tensor, inputs|
+        a, b = inputs
+        call_vector_op(:squared_difference, a, b, context, ->(t, u) { (t - u) * (t - u) })
+      end
+
       register_op :concat do |_context, tensor, inputs|
         concat_array(inputs[0], tensor.options[:axis])
       end
@@ -345,7 +386,7 @@ module TensorStream
           end
           reduced_val
         end
-
+   
         reduction(context, tensor, func)
       end
 
@@ -364,6 +405,25 @@ module TensorStream
         reduction(context, tensor, func)
       end
 
+      register_op :range do |context, tensor, inputs|
+        start, limit, delta = inputs
+        raise " delta !=0 " if delta.zero?
+        raise " Requires start <= limit when delta > 0" if (start > limit) && delta > 0
+        raise " Requires start >= limit when delta < 0" if (start < limit) && delta < 0
+        
+        
+        cur_step = start
+        r = []
+        Kernel.loop do
+          break if start == limit
+          break if (start < limit) && (cur_step >= limit)
+          break if (start > limit) && (cur_step <= limit)
+          r << cur_step
+          cur_step += delta
+        end
+        r
+      end
+
       register_op :tanh_grad, no_eval: true do |context, _tensor, inputs|
         call_op(:tanh_grad, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
       end
@@ -386,12 +446,13 @@ module TensorStream
         end
       end
 
-      register_op :cond do |context, tensor, inputs|
+      register_op :cond, noop: true do |context, tensor, inputs|
         pred = complete_eval(tensor.options[:pred], context)
+
         if all_true?(pred)
-          inputs[0]
+          complete_eval(inputs[0], context)
         else
-          inputs[1]
+          complete_eval(inputs[1], context)
         end
       end
 
@@ -420,25 +481,39 @@ module TensorStream
         call_vector_op(:greater_equal, a, b, context, ->(t, u) { t <= u })
       end
 
-      register_op %i[zeros ones zeros_like ones_like] do |_context, tensor, inputs|
-        shape = if %i[zeros_like ones_like].include?(tensor.operation)
-          shape_eval(inputs[0])
+      register_op :fill do |_context, tensor, inputs|
+        shape = inputs[0]
+        value = inputs[1]
+
+        func = -> { value }
+
+        if shape.is_a?(Array) && shape.size.zero?
+          func.call
         else
-          inputs[0] || tensor.shape.shape
+          shape = [shape.to_i] unless shape.is_a?(Array)
+          generate_vector(shape, generator: func)
         end
+      end
+
+      register_op %i[zeros ones zeros_like ones_like] do |_context, tensor, inputs|
+        shape = if %i[zeros_like ones_like].include?(tensor.operation)
+                  shape_eval(inputs[0])
+                else
+                  inputs[0] || tensor.shape.shape
+                end
 
         func = if %i[zeros zeros_like].include?(tensor.operation)
-                -> { tensor.data_type == :int32 ? 0 : 0.0 }
-              else
-                -> { tensor.data_type == :int32 ? 1 : 1.0 }
-              end
+                 -> { int_type?(tensor.data_type) ? 0 : 0.0 }
+               else
+                 -> { int_type?(tensor.data_type) ? 1 : 1.0 }
+               end
 
         if shape.is_a?(Array) && shape.size.zero?
           func.call
         else
           shape = [shape.to_i] unless shape.is_a?(Array)
 
-          cache_key = "#{tensor.operation}_#{shape.to_s}"
+          cache_key = "#{tensor.operation}_#{shape}"
           if @context[:_cache].key?(cache_key)
             @context[:_cache][cache_key]
           else
@@ -527,22 +602,10 @@ module TensorStream
         get_broadcast_gradient_args(inputs[0], inputs[1])
       end
 
-      register_op :reduced_shape do |context, _tensor, inputs|
-        input_shape, axes = inputs
-
-        return [] if axes.nil? # reduce to scalar
-        axes = [ axes ] unless axes.is_a?(Array)
-        return input_shape if axes.empty?
-
-        axes.each do |dimen|
-          input_shape[dimen] = 1
-        end
-        input_shape
-      end
-
-      register_op :tile do |context, _tensor, inputs|
+      register_op :tile do |_context, _tensor, inputs|
         input, multiples = inputs
         rank = get_rank(input)
+
         raise '1D or higher tensor required' if rank.zero?
         raise "invalid multiple size passed #{rank} != #{multiples.size}" if rank != multiples.size
 
@@ -554,21 +617,82 @@ module TensorStream
         inputs.collect { |input| run(input, context) }
       end
 
-      register_op :softmax do |context, _tensor, inputs|
+      register_op :softmax do |_context, _tensor, inputs|
         softmax(inputs[0])
       end
 
       register_op :softmax_grad do |_context, _tensor, inputs|
         input, grad = inputs
-
         softmax_input = softmax(input)
-        f_grad = softmax_grad(softmax_input)
-        f_grad.transpose.each_with_index.collect do |row, index|
-          sum = 0.0
-          row.each_with_index do |r, g_index|
-            sum += r * grad[g_index]
+        input_shape = shape_eval(input)
+
+        last_dimen_list = last_axis(softmax_input)
+        last_grad_list = last_axis(grad)
+
+        func = -> (list, last_grad) {
+
+          f_grad = softmax_grad(list)
+          f_grad.transpose.each_with_index.collect do |row, index|
+            sum = 0.0
+            row.each_with_index do |r, g_index|
+              sum += r * last_grad[g_index]
+            end
+            sum
+          end
+        }
+
+        if input_shape.size == 1
+          func.(last_dimen_list, last_grad_list)
+        else
+          arr = last_dimen_list.zip(last_grad_list).collect do |list, last_grad|
+            func.(list, last_grad)
           end
-          sum
+          TensorShape.reshape(arr.flatten, input_shape)
+        end
+        
+      end
+
+      register_op :softmax_cross_entropy_with_logits_v2 do |context, tensor, inputs|
+        last_dimen_list = last_axis(inputs[0])
+        input_shape = shape_eval(inputs[0])
+        labels = last_axis(inputs[1])
+        func = -> (logits, label) {
+          c = logits.max
+          transformed_logits = logits.map { |l| l - c}
+          sum = transformed_logits.map { |x| Math.exp(x) }.reduce(:+)
+          transformed_logits.zip(label).map { |x, y| (Math.log(sum) - x) * y }
+        }
+
+        if input_shape.size == 1
+          func.(last_dimen_list, labels)
+        else
+          arr = last_dimen_list.zip(labels).collect do |list, label|
+            func.(list, label)
+          end
+          TensorShape.reshape(arr.flatten, input_shape)
+        end
+      end
+
+      register_op :softmax_cross_entropy_with_logits_v2_grad do |context, tensor, inputs|
+        last_dimen_list = last_axis(inputs[0])
+        labels = last_axis(inputs[1])
+        passed_grads = last_axis(inputs[2])
+        input_shape = shape_eval(inputs[0])
+
+        func = -> (logits, label, grad) {
+          c = logits.max
+          transformed_logits = logits.map { |l| Math.exp(l - c) }
+          e_sum = transformed_logits.reduce(:+)
+          transformed_logits.zip(label).zip(grad).map { |(x, y), g|  (x / e_sum) * g  - y }
+        }
+
+        if input_shape.size == 1
+          func.(last_dimen_list, labels, passed_grads)
+        else
+          arr = last_dimen_list.zip(labels).zip(passed_grads).collect do | (list, label), passed_grad|
+            func.(list, label, passed_grad)
+          end
+          TensorShape.reshape(arr.flatten, input_shape)
         end
       end
 
@@ -580,9 +704,13 @@ module TensorStream
 
       def eval_operation(tensor, child_context)
         return @context[tensor.name] if @context.key?(tensor.name)
-
-        # puts tensor.name
         invoke(tensor, child_context).tap do |result|
+          # puts tensor.name
+          # if tensor.name.start_with?('softmax_cross_entropy_with_logits')
+          #   puts result.inspect
+          # end
+          # result.flatten.each do |a|
+          # end if result.is_a?(Array)
           if tensor.breakpoint
             a = resolve_placeholder(tensor.inputs[0], child_context) if tensor.inputs && tensor.inputs[0]
             b = resolve_placeholder(tensor.inputs[1], child_context) if tensor.inputs && tensor.inputs[1]
@@ -792,17 +920,6 @@ module TensorStream
         shape.is_a?(Array) ? shape.size : 0
       end
 
-      def get_broadcast_gradient_args(input_a, input_b)
-        return [] if get_rank(input_b).zero? && get_rank(input_a).zero?
-        return nil if get_rank(input_b).zero?
-        # ruby scalar
-        if get_rank(input_a).zero?
-          _broadcast_gradient_op(input_b, input_a, 0, true)
-        elsif get_rank(input_a) > 0
-          _broadcast_gradient_op(input_a, input_b, 0)
-        end
-      end
-
       def concat_array(values, axis)
         combined_array = values.shift
         axis = get_rank(combined_array) - 1 if axis == -1

data/lib/tensor_stream/graph_serializers/pbtext.rb

@@ -40,11 +40,19 @@ module TensorStream
     private
 
     def process_options(node)
+      return if node.options.nil?
       node.options.each do |k, v|
         next if %w[name].include?(k.to_s)
         @lines << "  attr {"
         @lines << "    key: \"#{k}\""
         @lines << "    value {"
+        if (v.is_a?(TrueClass) || v.is_a?(FalseClass))
+          @lines << "       b: #{v.to_s}"
+        elsif (v.is_a?(Integer))
+          @lines << "       int_val: #{v}"
+        elsif (v.is_a?(Float))
+          @lines << "       float_val: #{v}"
+        end
         @lines << "    }"
         @lines << "  }"
       end

data/lib/tensor_stream/helpers/op_helper.rb

@@ -1,8 +1,8 @@
 module TensorStream
   # module that contains helper functions useful for ops
   module OpHelper
-    def _op(code, t_a, t_b = nil, options = {})
-      op = Operation.new(code.to_sym, t_a, t_b, options)
+    def _op(code, *args)
+      op = Operation.new(code.to_sym, *args)
       if !TensorStream.get_default_graph.get_dependency_scope.nil?
         i_op(:identity, op, TensorStream.get_default_graph.get_dependency_scope, name: [op.name, 'tuple', 'control_dependency'].join('/'))
       else
@@ -11,8 +11,15 @@ module TensorStream
     end
 
     # same as op but with a marker that it was internal generated
-    def i_op(code, t_a, t_b = nil, options = {})
-      Operation.new(code.to_sym, t_a, t_b, options.merge(internal: true))
+    def i_op(code, *args)
+      options = if args.last.is_a?(Hash)
+        args.pop
+      else
+        {}
+      end
+
+      args << options.merge(internal: true)
+      Operation.new(code.to_sym, *args)
     end
 
     def cons(value, options = {})
@@ -68,10 +75,40 @@ module TensorStream
       TensorStream::Ops::FLOATING_POINT_TYPES.include?(type)
     end
 
+    def int_type?(type)
+      TensorStream::Ops::INTEGER_TYPES.include?(type)
+    end
+
     def format_source(trace)
       grad_source = trace.select { |c| c.to_s.include?(File.join('lib', 'tensor_stream', 'math_gradients')) }.first
       source = trace.reject { |c| c.to_s.include?(File.join('lib', 'tensor_stream')) }.first
       [grad_source, source].compact.join("\n")
     end
+
+    def shapes_fully_specified_and_equal(x, y)
+      return false if !shape_full_specified(x) || !shape_full_specified(y)
+      return false if x.shape.shape != y.shape.shape
+      
+      true
+    end
+ 
+    def shape_full_specified(tensor)
+      return false if tensor.shape.nil?
+      return false if tensor.shape.shape.nil?
+
+      tensor.shape.shape.each { |s| return false if s.nil? }
+      true
+    end
+
+    def reduced_shape(input_shape, axes)
+      input_shape = TensorStream.convert_to_tensor(input_shape)
+      axes = TensorStream.convert_to_tensor(axes)
+      input_rank = i_op(:size, input_shape)
+      axes = TensorStream.range(0, input_rank) if axes.nil?
+      axes = (axes + input_rank) % input_rank
+      axes_shape = i_op(:shape, axes)
+      TensorStream.dynamic_stitch([TensorStream.range(0, input_rank), axes],
+        [input_shape, i_op(:fill, axes_shape, 1)])
+    end
   end
 end