tensor_stream 0.1.4 → 0.1.5

data/lib/tensor_stream/nn/nn_ops.rb

@@ -8,6 +8,23 @@ module TensorStream
     def self.relu(features, name: nil)
       TensorStream.max(features, 0, name: "relu_#{name}")
     end
+
+    def self.sigmoid_cross_entropy_with_logits(labels: nil, logits: nil, name: nil)
+      TensorStream.name_scope(name, default: 'logistic_loss', values: [logits, labels]) do |name|
+        tf = TensorStream
+        logits = tf.convert_to_tensor(logits, name: 'logits')
+        labels = tf.convert_to_tensor(labels, name: 'labels')
+        zeros = tf.zeros_like(logits, dtype: logits.dtype)
+        cond = (logits >= zeros)
+        relu_logits = tf.where(cond, logits, zeros)
+        neg_abs_logits = tf.where(cond, -logits, logits)
+
+        return tf.add(
+            relu_logits - logits * labels,
+            tf.log1p(tf.exp(neg_abs_logits)),
+            name: name)
+      end
+    end
   end
 
   # tensorflow compatibility

data/lib/tensor_stream/operation.rb

@@ -9,18 +9,17 @@ module TensorStream
 
       @operation = operation
       @rank = options[:rank] || 0
-      @name = options[:name] || set_name
+      @name = [@graph.get_name_scope, options[:name] || set_name].compact.join('/')
       @internal = options[:internal]
       @given_name = @name
       @source = format_source(caller_locations)
 
       @options = options
 
-      @items = [input_a, input_b].map { |i| options[:preserve_params_type] ? i : auto_wrap(i) }
+      @items = [input_a, input_b].map { |i| options[:preserve_params_type] ? i : TensorStream.convert_to_tensor(i) }
       @data_type = set_data_type(options[:data_type])
 
-      @shape = TensorShape.new(options[:shape], options[:shape].size || 0) if options[:shape]
-
+      @shape = TensorShape.new(infer_shape)
       @graph.add_node(self)
     end
 
@@ -57,47 +56,55 @@ module TensorStream
       when :shape, :rank
         :int32
       else
-        passed_data_type || (@items[0] ? @items[0].data_type : :unknown)
+        return passed_data_type if passed_data_type
+        if @items[0]
+          @items[0].data_type
+        elsif @items[1]
+          @items[1].data_type
+        else
+          :unknown
+        end
       end
     end
 
-    def to_math(name_only = false, max_depth = 99)
+    def to_math(name_only = false, max_depth = 99, _cur_depth = 0)
       return @name if max_depth.zero?
 
-      sub_item = auto_math(items[0], name_only, max_depth - 1)
+      sub_item = auto_math(items[0], name_only, max_depth - 1, _cur_depth + 1)
+      sub_item2 = auto_math(items[1], name_only, max_depth - 1, _cur_depth + 1) if items[1]
 
-      case operation
+      out = case operation
       when :argmax
-        "argmax(#{auto_math(items[0])},#{options[:axis]})"
+        "argmax(#{sub_item},#{options[:axis]})"
       when :negate
         "-#{sub_item}"
       when :index
-        "#{sub_item}[#{auto_math(items[1], name_only, max_depth - 1)}]"
+        "#{sub_item}[#{sub_item2}]"
       when :slice
-        "#{sub_item}[#{auto_math(items[1], name_only, max_depth - 1)}]"
+        "#{sub_item}[#{sub_item2}]"
       when :assign_sub
-        "(#{items[0] ? items[0].name : 'self'} -= #{auto_math(items[1], name_only)})"
+        "(#{items[0] ? items[0].name : 'self'} -= #{auto_math(items[1], name_only, 1)})"
       when :assign_add
-        "(#{items[0] ? items[0].name : 'self'} += #{auto_math(items[1], name_only)})"
+        "(#{items[0] ? items[0].name : 'self'} += #{auto_math(items[1], name_only, 1)})"
       when :assign
-        "(#{items[0] ? items[0].name : 'self'} = #{auto_math(items[1], name_only)})"
+        "(#{items[0] ? items[0].name : 'self'} = #{auto_math(items[1], name_only, 1)})"
       when :sin, :cos, :tanh
         "#{operation}(#{sub_item})"
       when :add
-        "(#{sub_item} + #{auto_math(items[1], name_only, max_depth - 1)})"
+        "(#{sub_item} + #{sub_item2})"
       when :sub
-        "(#{sub_item} - #{auto_math(items[1], name_only, max_depth - 1)})"
+        "(#{sub_item} - #{sub_item2})"
       when :pow
-        "(#{sub_item}^#{auto_math(items[1], name_only, max_depth - 1)})"
+        "(#{sub_item}^#{sub_item2})"
       when :div
-        "(#{sub_item} / #{auto_math(items[1], name_only, max_depth - 1)})"
+        "(#{sub_item} / #{sub_item2})"
       when :mul
         if auto_math(items[0]) == 1
-          auto_math(items[1], name_only, max_depth - 1)
+          sub_item2
         elsif auto_math(items[1]) == 1
           sub_item
         else
-          "(#{sub_item} * #{auto_math(items[1], name_only, max_depth - 1)})"
+          "(#{sub_item} * #{sub_item2})"
         end
       when :reduce_sum
         "reduce_sum(|#{sub_item}|)"
@@ -110,7 +117,7 @@ module TensorStream
       when :stop_gradient
         sub_item
       when :matmul
-        "#{sub_item}.matmul(#{auto_math(items[1], name_only, max_depth - 1)})"
+        "#{sub_item}.matmul(#{sub_item2})"
       when :eye
         "eye(#{sub_item})"
       when :transpose
@@ -128,15 +135,19 @@ module TensorStream
       when :zeros
         "zeros(#{sub_item})"
       when :reshape
-        "reshape(#{sub_item},#{auto_math(items[1], name_only, max_depth - 1)})"
+        "reshape(#{sub_item},#{sub_item2})"
       when :rank
         "#{sub_item}.rank"
       when :cond
-        "(#{auto_math(options[:pred])} ? #{sub_item} : #{auto_math(items[1], name_only, max_depth - 1)})"
+        "(#{auto_math(options[:pred], name_only, max_depth - 1, _cur_depth)} ? #{sub_item} : #{sub_item2})"
       when :less
-        "#{sub_item} < #{auto_math(items[1], name_only, max_depth - 1)}"
+        "#{sub_item} < #{sub_item2}"
+      when :less_equal
+        "#{sub_item} <= #{sub_item2}"
       when :greater
-        "#{sub_item} > #{auto_math(items[1], name_only, max_depth - 1)}"
+        "#{sub_item} > #{sub_item2}"
+      when :greater_equal
+        "#{sub_item} >= #{sub_item2}"
       when :square
         "#{sub_item}\u00B2"
       when :log
@@ -148,32 +159,82 @@ module TensorStream
       when :pad
         "pad(#{sub_item},#{auto_math(options[:paddings])})"
       when :equal
-        "#{sub_item} == #{auto_math(items[1], name_only, max_depth - 1)}"
+        "#{sub_item} == #{sub_item2}"
       when :not_equal
-        "#{sub_item} != #{auto_math(items[1], name_only, max_depth - 1)}"
+        "#{sub_item} != #{sub_item2}"
       when :logical_and
-        "#{sub_item} && #{auto_math(items[1], name_only, max_depth - 1)}"
+        "#{sub_item} && #{sub_item2}"
       when :sqrt
         "sqrt(#{sub_item})"
       when :zeros_like
         "zeros_like(#{sub_item})"
       when :where
-        "where(#{auto_math(options[:pred], name_only, max_depth - 1)},#{auto_math(items[0])},#{auto_math(items[1])})"
+        "where(#{auto_math(options[:pred], name_only, max_depth - 1, _cur_depth)}, #{sub_item}, #{sub_item2})"
       when :max
-        "max(#{auto_math(sub_item)},#{auto_math(items[1])})"
+        "max(#{sub_item},#{sub_item2})"
       when :cast
-        "cast(#{auto_math(sub_item)}, #{data_type})"
+        "cast(#{sub_item}, #{data_type})"
       else
         raise "no math form for #{operation} defined"
       end
+      ["\n",(_cur_depth + 1).times.collect { ' ' }, out].flatten.join
     end
 
     def run
       eval
     end
 
+    def op
+      self
+    end
+
     private
 
+    def infer_shape
+      case operation
+      when :index
+        item_shape = items[0].shape.shape
+        return nil if item_shape.nil?
+        return item_shape[1, item_shape.size]
+      when :reduce_mean, :reduce_prod, :reduce_sum
+        return [] if options[:axis].nil?
+        item_shape = items[0].shape.shape
+        return nil if item_shape.nil?
+        axis = options[:axis]
+
+        axis = [ axis ] unless axis.is_a?(Array)
+        return item_shape.each_with_index.map do |s, index|
+          next nil if axis.include?(index)
+          s
+        end.compact
+      when :reshape
+        new_shape = items[1] && items[1].value ? items[1].value : nil
+        return nil if new_shape.nil?
+
+        item_shape = items[0].shape.shape
+        return new_shape if item_shape.nil?
+
+        return TensorShape.fix_inferred_elements(new_shape, item_shape.reduce(:*))
+      when :flow_group
+        return []
+      when :zeros, :ones
+        return items[0] ? items[0].value : options[:shape]
+      when :shape
+        return items[0].shape.shape ? [items[0].shape.shape.size] : nil
+      when :matmul
+        shape1 = items[0].shape.shape.nil? ? nil : items[0].shape.shape[0]
+        shape2 = items[1].shape.shape.nil? ? nil : items[1].shape.shape[1]
+        return [shape1, shape2]
+      else
+        return items[0].shape.shape if items.size == 1
+        if items.size == 2 && items[0] && items[1]
+          return TensorShape.infer_shape(items[0].shape.shape, items[1].shape.shape)
+        end
+      end
+
+      nil
+    end
+
     def propagate_consumer(consumer)
       super(consumer)
 

data/lib/tensor_stream/ops.rb

@@ -5,7 +5,7 @@ module TensorStream
     NUMERIC_TYPES = %w[int32 int64 float32 float64].map(&:to_sym)
 
     def argmax(input, axis = nil, name: nil, dimension: nil, output_type: :int32)
-      op(:argmax, input, nil, axis: axis, name: name, dimension: dimension, data_type: output_type)
+      _op(:argmax, input, nil, axis: axis, name: name, dimension: dimension, data_type: output_type)
     end
 
     def gradients(input, wrt_xs, grad_ys: nil,
@@ -24,10 +24,12 @@ module TensorStream
         tensor_program = if input.graph.node_added?(gradient_program_name)
                            input.graph.get_node(gradient_program_name)
                          else
-                           derivative_ops = TensorStream::MathGradients.derivative(input, x, graph: input.graph,
-                                                                                             stop_gradients: stop_gradients)
-                           unit_matrix = op(:ones_like, x)
-                           input.graph.add_node!(gradient_program_name, unit_matrix * derivative_ops)
+                          input.graph.name_scope("gradient_wrt_#{x.name}") do
+                            derivative_ops = TensorStream::MathGradients.derivative(input, x, graph: input.graph,
+                                                                                              stop_gradients: stop_gradients)
+                            unit_matrix = _op(:ones_like, x)
+                            input.graph.add_node!(gradient_program_name, unit_matrix * derivative_ops)
+                           end
                          end
         tensor_program
       end
@@ -36,183 +38,204 @@ module TensorStream
 
     def random_uniform(shape, dtype: :float32, minval: 0, maxval: 1, seed: nil, name: nil)
       options = { shape: shape, dtype: dtype, minval: minval, maxval: maxval, seed: seed, name: name }
-      op(:random_uniform, nil, nil, options)
+      _op(:random_uniform, nil, nil, options)
     end
 
     def random_normal(shape, dtype: :float32, mean: 0.0, stddev: 1.0, seed: nil, name: nil)
       options = { shape: shape, dtype: dtype, mean: mean, stddev: stddev, seed: seed, name: name }
-      op(:random_normal, nil, nil, options)
+      _op(:random_normal, nil, nil, options)
     end
 
     def stop_gradient(tensor, options = {})
-      op(:stop_gradient, tensor, nil, options)
+      _op(:stop_gradient, tensor, nil, options)
     end
 
     def eye(num_rows, num_columns: nil, dtype: :float32, name: nil)
-      op(:eye, num_rows, num_columns || num_rows, data_type: dtype, name: name)
+      _op(:eye, num_rows, num_columns || num_rows, data_type: dtype, name: name)
     end
 
     def shape(input, name: nil, out_type: :int32)
-      op(:shape, input, nil, name: name, out_type: out_type)
+      _op(:shape, input, nil, name: name, out_type: out_type)
     end
 
     def rank(input, name: nil)
-      op(:rank, input, name: name)
+      _op(:rank, input, name: name)
     end
 
     def zeros_initializer(options = {})
-      op(:zeros, nil, nil, options)
+      _op(:zeros, nil, nil, options)
+    end
+
+    def glorot_uniform_initializer(seed: nil, dtype: :float32)
+      TensorStream::Initializer.new(-> { _op(:glorot_uniform, nil, nil, seed: seed, data_type: dtype) })
+    end
+
+    def random_uniform_initializer(minval: 0, maxval: 1, seed: nil, dtype: nil)
+      TensorStream::Initializer.new(-> { _op(:random_uniform, nil, nil, minval: 0, maxval: 1, seed: seed, data_type: dtype) })
     end
 
     def slice(input, start, size, name: nil)
-      op(:slice, input, start, size: size, name: name)
+      _op(:slice, input, start, size: size, name: name)
     end
 
     def zeros(shape, dtype: :float32, name: nil)
-      op(:zeros, shape, nil, data_type: dtype, name: name)
+      _op(:zeros, shape, nil, data_type: dtype, name: name)
     end
 
     def ones(shape, dtype: :float32, name: nil)
-      op(:ones, shape, nil, data_type: dtype, name: name)
+      _op(:ones, shape, nil, data_type: dtype, name: name)
     end
 
     def less(input_a, input_b, name: nil)
-      op(:less, input_a, input_b, name: name)
+      _op(:less, input_a, input_b, name: name)
     end
 
     def logical_and(input_a, input_b, name: nil)
-      op(:logical_and, input_a, input_b, name: name)
+      _op(:logical_and, input_a, input_b, name: name)
     end
 
     def greater(input_a, input_b, name: nil)
-      op(:greater, input_a, input_b, name: name)
+      _op(:greater, input_a, input_b, name: name)
     end
 
     def greater_equal(input_a, input_b, name: nil)
-      op(:greater_equal, input_a, input_b, name: name)
+      _op(:greater_equal, input_a, input_b, name: name)
     end
 
     def less_equal(input_a, input_b, name: nil)
-      op(:less_equal, input_a, input_b, name: name)
+      _op(:less_equal, input_a, input_b, name: name)
     end
 
     def reduce_mean(input_tensor, axis = nil, keepdims: false, name: nil)
-      op(:reduce_mean, input_tensor, nil, axis: axis, keepdims: keepdims, name: name)
+      _op(:reduce_mean, input_tensor, nil, axis: axis, keepdims: keepdims, name: name)
     end
 
     def reduce_sum(input_tensor, axis = nil, keepdims: false, name: nil)
-      op(:reduce_sum, input_tensor, nil, axis: axis, keepdims: keepdims, name: name)
+      _op(:reduce_sum, input_tensor, nil, axis: axis, keepdims: keepdims, name: name)
     end
 
     def reduce_prod(input, axis = nil, keepdims: false, name: nil)
-      op(:reduce_prod, input, nil, axis: axis, keepdims: keepdims, name: name)
+      _op(:reduce_prod, input, nil, axis: axis, keepdims: keepdims, name: name)
     end
 
     def concat(values, axis, name: 'concat')
-      op(:concat, values, nil, axis: axis, name: name)
+      _op(:concat, values, nil, axis: axis, name: name)
     end
 
     def reshape(tensor, shape, name: nil)
-      op(:reshape, tensor, shape, name: name)
+      _op(:reshape, tensor, shape, name: name)
     end
 
     def square(tensor, name: nil)
-      op(:square, tensor, nil, name: name)
+      _op(:square, tensor, nil, name: name)
+    end
+
+    def round(tensor, name: nil)
+      check_allowed_types(tensor, FLOATING_POINT_TYPES)
+      _op(:round, tensor, nil, name: name)
+    end
+
+    def reciprocal(tensor, name: nil)
+      _op(:reciprocal, tensor, nil, name: name)
     end
 
     def cond(pred, true_fn, false_fn, name: nil)
-      op(:cond, true_fn, false_fn, pred: pred, name: name)
+      _op(:cond, true_fn, false_fn, pred: pred, name: name)
     end
 
     def where(condition, true_t = nil, false_t = nil, name: nil)
-      op(:where, true_t, false_t, pred: condition, name: name)
+      _op(:where, true_t, false_t, pred: condition, name: name)
     end
 
     def add(input_a, input_b, name: nil)
-      op(:add, input_a, input_b, name: name)
+      _op(:add, input_a, input_b, name: name)
     end
 
     def sub(input_a, input_b, name: nil)
-      op(:sub, input_a, input_b, name: name)
+      _op(:sub, input_a, input_b, name: name)
     end
 
     def max(input_a, input_b, name: nil)
       check_allowed_types(input_a, NUMERIC_TYPES)
       check_allowed_types(input_b, NUMERIC_TYPES)
 
-      op(:max, input_a, input_b, name: name)
+      _op(:max, input_a, input_b, name: name)
     end
 
+    def maximum(input_a, input_b, name: nil)
+      max(input_a, input_b, name: name)
+    end
+  
     def cast(input, dtype, name: nil)
-      op(:cast, input, nil, data_type: dtype, name: name)
+      _op(:cast, input, nil, data_type: dtype, name: name)
     end
 
     def print(input, data, message: nil, name: nil)
-      op(:print, input, data, message: message, name: name)
+      _op(:print, input, data, message: message, name: name)
     end
 
     def negate(input, options = {})
-      op(:negate, input, nil, options)
+      _op(:negate, input, nil, options)
     end
 
     def equal(input_a, input_b, name: nil)
-      op(:equal, input_a, input_b, name: name)
+      _op(:equal, input_a, input_b, name: name)
     end
 
     def not_equal(input_a, input_b, name: nil)
-      op(:not_equal, input_a, input_b, name: name)
+      _op(:not_equal, input_a, input_b, name: name)
     end
 
     def zeros_like(tensor, dtype: nil, name: nil)
-      op(:zeros_like, tensor, nil, data_type: dtype, name: name)
+      _op(:zeros_like, tensor, nil, data_type: dtype, name: name)
     end
 
     def ones_like(tensor, dtype: nil, name: nil)
-      op(:ones_like, tensor, nil, data_type: dtype, name: name)
+      _op(:ones_like, tensor, nil, data_type: dtype, name: name)
     end
 
     def identity(input, name: nil)
-      op(:identity, input, nil, name: name)
+      _op(:identity, input, nil, name: name)
     end
 
     def multiply(input_a, input_b, name: nil)
-      op(:mul, input_a, input_b, name: name)
+      _op(:mul, input_a, input_b, name: name)
     end
 
     def pow(input_a, input_e, name: nil)
-      op(:pow, input_a, input_e, name: name)
+      _op(:pow, input_a, input_e, name: name)
     end
 
     def abs(input, name: nil)
-      op(:abs, input, nil, name: name)
+      _op(:abs, input, nil, name: name)
     end
 
     def sign(input, name: nil)
-      op(:sign, input, nil, name: name)
+      _op(:sign, input, nil, name: name)
     end
 
     def sin(input, options = {})
       options[:data_type] ||= :float32
       check_allowed_types(input, FLOATING_POINT_TYPES)
-      op(:sin, input, nil, options)
+      _op(:sin, input, nil, options)
     end
 
     def cos(input, options = {})
       options[:data_type] ||= :float32
       check_allowed_types(input, FLOATING_POINT_TYPES)
-      op(:cos, input, nil, options)
+      _op(:cos, input, nil, options)
     end
 
     def tan(input, options = {})
       options[:data_type] ||= :float32
       check_allowed_types(input, FLOATING_POINT_TYPES)
-      op(:tan, input, nil, options)
+      _op(:tan, input, nil, options)
     end
 
     def tanh(input, options = {})
       options[:data_type] ||= :float32
       check_allowed_types(input, FLOATING_POINT_TYPES)
-      op(:tanh, input, nil, options)
+      _op(:tanh, input, nil, options)
     end
 
     def sqrt(input, name: nil)
@@ -221,33 +244,44 @@ module TensorStream
         name: name
       }
       check_allowed_types(input, FLOATING_POINT_TYPES)
-      op(:sqrt, input, nil, options)
+      _op(:sqrt, input, nil, options)
     end
 
     def log(input, options = {})
       options[:data_type] ||= :float32
       check_allowed_types(input, FLOATING_POINT_TYPES)
-      op(:log, input, nil, options)
+      _op(:log, input, nil, options)
+    end
+      
+    def log1p(input, options = {})
+      options[:data_type] ||= :float32
+      check_allowed_types(input, FLOATING_POINT_TYPES)
+      _op(:log1p, input, nil, options)
     end
 
     def exp(input, options = {})
       options[:data_type] ||= :float32
       check_allowed_types(input, FLOATING_POINT_TYPES)
-      op(:exp, input, nil, options)
+      _op(:exp, input, nil, options)
+    end
+
+    def sigmoid(input, name: nil)
+      check_allowed_types(input, FLOATING_POINT_TYPES)
+      _op(:sigmoid, input, nil, name: name)
     end
 
     def matmul(input_a, input_b, transpose_a: false,
                transpose_b: false,
                name: nil)
-      op(:matmul, input_a, input_b, transpose_a: transpose_a, transpose_b: transpose_b, name: name)
+      _op(:matmul, input_a, input_b, transpose_a: transpose_a, transpose_b: transpose_b, name: name)
     end
 
     def transpose(tensor, perm: nil, name: 'transpose')
-      op(:transpose, tensor, nil, perm: perm, name: name)
+      _op(:transpose, tensor, nil, perm: perm, name: name)
     end
 
     def pad(tensor, paddings, mode: 'CONSTANT', name: nil)
-      op(:pad, tensor, nil, paddings: paddings, mode: mode, name: name)
+      _op(:pad, tensor, nil, paddings: paddings, mode: mode, name: name)
     end
   end
 end