tensor_stream 0.6.1 → 0.7.0

data/lib/tensor_stream/session.rb

@@ -115,6 +115,10 @@ module TensorStream
     def delegate_to_evaluator(tensor_arr, session_context, context)
       arr = tensor_arr.is_a?(Array) ? tensor_arr : [tensor_arr]
       result = arr.collect do |tensor|
+        if session_context[:_cache][:placement][tensor.name].nil?
+          session_context[:_cache][:placement][tensor.name] = assign_evaluator(tensor)
+        end
+
         session_context[:_cache][:placement][tensor.name][1].run_with_buffer(tensor, session_context, context)
       end
       result.size == 1 ? result.first : result

data/lib/tensor_stream/tensor.rb

@@ -100,6 +100,10 @@ module TensorStream
       TensorStream.ceil(self)
     end
 
+    def zero?
+      _op(:equal, self, TensorStream.constant(0, dtype: data_type, name: 'equal/is_zero?'))
+    end
+
     def ==(other)
       _a, other = TensorStream.check_data_types(self, other)
       _op(:equal, self, other)
@@ -137,12 +141,28 @@ module TensorStream
 
     def matmul(other)
       _a, other = TensorStream.check_data_types(self, other)
-      _op(:matmul, self, other)
+      _op(:mat_mul, self, other)
     end
 
     def dot(other)
       _a, other = TensorStream.check_data_types(self, other)
-      _op(:matmul, self, other)
+      _op(:mat_mul, self, other)
+    end
+
+    ##
+    # Apply a reduction to tensor
+    def reduce(op_type)
+      reduce_op = case op_type.to_sym
+                  when :+
+                    :sum
+                  when :*
+                    :prod
+                  else
+                    raise "unsupported reduce op type #{op_type}"
+                  end
+      raise "blocks are not supported for tensors" if block_given?
+
+      _op(reduce_op, self, nil)
     end
 
     def collect(&block)
@@ -154,7 +174,7 @@ module TensorStream
     end
 
     def op
-      is_const ? _op(:const, self, nil, name: self.name) : _op(:variable, self, nil, name: self.name)
+      is_const ? _op(:const, self, nil, name: name) : _op(:variable, self, nil, name: name)
     end
 
     def eval(options = {})
@@ -197,12 +217,12 @@ module TensorStream
       end
     end
 
-    def auto_math(tensor, name_only = false, max_depth = 99, _cur_depth = 0)
-      tensor.is_a?(Tensor) ? tensor.to_math(name_only, max_depth, _cur_depth) : tensor
+    def auto_math(tensor, name_only = false, max_depth = 99, cur_depth = 0)
+      tensor.is_a?(Tensor) ? tensor.to_math(name_only, max_depth, cur_depth) : tensor
     end
 
     def self.detect_type(value)
-      if !!value==value
+      if !!value == value
         :boolean
       elsif value.is_a?(String)
         :string
@@ -211,7 +231,7 @@ module TensorStream
       elsif value.is_a?(Integer)
         :int32
       elsif value.is_a?(Array)
-        return detect_type(value[0])
+        detect_type(value[0])
       elsif value.is_a?(Tensor)
         value.data_type
       else
@@ -229,9 +249,7 @@ module TensorStream
         end
       end
 
-      if dtype.is_a?(Hash)
-        dtype = dtype[:dtype]
-      end
+      dtype = dtype[:dtype] if dtype.is_a?(Hash)
 
       case dtype.to_sym
       when :float64, :float32, :float
@@ -257,7 +275,7 @@ module TensorStream
       end
     end
 
-    def breakpoint!(&block)
+    def breakpoint!(&_block)
       self
     end
 
@@ -269,7 +287,7 @@ module TensorStream
 
     def setup_initial_state(options)
       @outputs = []
-      @graph = options[:graph] || TensorStream.get_default_graph
+      @graph = options[:__graph] || TensorStream.get_default_graph
       @source = format_source(caller_locations)
     end
 

data/lib/tensor_stream/tensor_shape.rb

@@ -71,7 +71,7 @@ module TensorStream
       return shape if shape.empty?
 
       current_size = shape.inject(1) { |product, n| n > 0 ? product * n : product }
-      inferred_size = total_size / current_size
+      inferred_size = total_size.nil? ? nil : total_size / current_size
       shape.map { |s| s == -1 ? inferred_size : s }
     end
   end

data/lib/tensor_stream/train/gradient_descent_optimizer.rb

@@ -2,17 +2,50 @@ module TensorStream
   module Train
     # High Level implementation of the gradient descent algorithm
     class GradientDescentOptimizer
+      include TensorStream::OpHelper
+
       attr_accessor :learning_rate
 
       def initialize(learning_rate, _options = {})
         @learning_rate = learning_rate
       end
 
-      def minimize(cost)
-        trainable_vars = TensorStream.trainable_variables
-        derivatives = TensorStream.gradients(cost, trainable_vars)
+      def minimize(loss, var_list: nil, grad_loss: nil, global_step: nil)
+        grads_and_vars = compute_gradients(loss, var_list: var_list, grad_loss: grad_loss)
+        apply_gradients(grads_and_vars, global_step: global_step)
+      end
+
+      ##
+      # Apply gradients to variables.
+      # This is the second part of minimize(). It returns an Operation that applies gradients.
+      def apply_gradients(grads_and_vars, global_step: nil)
+        apply_ops = grads_and_vars.map do |grad, var|
+          i_op(:apply_gradient_descent, var, TensorStream.cast(@learning_rate, grad.data_type), grad)
+        end
+
+        if global_step.nil?
+          apply_ops
+        else
+          apply_ops + [global_step.assign_add(1)]
+        end
+      end
+
+      ##
+      # Compute gradients of loss for the variables in var_list.
+      #
+      # This is the first part of minimize(). It returns a list of (gradient, variable) pairs where "gradient" is the gradient for "variable".
+      def compute_gradients(loss, var_list: nil, grad_loss: nil)
+        trainable_vars = if var_list
+                           raise "var_list must be an array" unless var_list.is_a?(Array)
+                           var_list.each_with_index { |var, index| raise "var #{index} not a Variable" unless var.is_a?(Variable) }
+
+                           var_list
+                         else
+                           loss.graph.get_collection(TensorStream::GraphKeys::TRAINABLE_VARIABLES)
+                         end
+        all_grads = grad_loss || TensorStream.gradients(loss, trainable_vars)
         trainable_vars.each_with_index.collect do |var, index|
-          var.assign_sub(derivatives[index] * @learning_rate)
+          [all_grads[index], var]
         end
       end
     end

data/lib/tensor_stream/train/saver.rb

@@ -4,6 +4,8 @@ module TensorStream
   module Train
     # High level class used for loading and saving variables
     class Saver
+      include TensorStream::OpHelper
+
       def save(session, outputfile, global_step: nil,
                latest_filename: nil,
                meta_graph_suffix: 'meta',
@@ -45,6 +47,50 @@ module TensorStream
 
       private
 
+      def build_internal(names_to_saveables, reshape: false, sharded: false, max_to_keep: 5,
+        keep_checkpoint_every_n_hours: 10000.0,
+        name: nil,
+        restore_sequentially: false,
+        filename: "model",
+        build_save: true,
+        build_restore: true)
+        saveables = _validate_and_slice_inputs(names_to_saveables)
+
+      end
+
+      def _validate_and_slice_inputs(names_to_saveables)
+        saveables = []
+        seen_ops = []
+
+        names_to_saveables.values.sort_by { |item| item[0] }.each do |name, op|
+          _saveable_objects_for_op(op, name).each do |converted_saveable_object|
+            _add_saveable(saveables, seen_ops, converted_saveable_object)
+          end
+        end
+        saveables
+      end
+
+      def _add_saveable(saveables, seen_ops, saveable)
+        raise TensorStreamm::ValueError, "The same saveable will be restored with two names: #{saveable.name}" if seen_ops.include?(saveable.op)
+        saveables << saveable
+        seen_ops << saveable.op
+      end
+
+      def save_op(filename_tensor, saveables)
+        tensor_names = []
+        tensors = []
+        tensor_slices = []
+
+        saveables.each do |saveable|
+          saveable.specs.each do |spec|
+            tensor_names << spec.name
+            tensors << spec.tensor
+            tensor_slices << spec.slice_spec
+          end
+        end
+        i_op(:save_v2, filename_tensor, tensor_names, tensor_slices, tensors)
+      end
+
       def eval_global_step(session, global_step)
         return nil if global_step.nil?
 

data/lib/tensor_stream/train/utils.rb

@@ -0,0 +1,37 @@
+module TensorStream
+  module Train
+    # convenience methods used for training
+    module Utils
+      def create_global_step(graph = nil)
+        target_graph = graph || TensorStream.get_default_graph
+        raise TensorStream::ValueError, '"global_step" already exists.' unless get_global_step(target_graph).nil?
+
+        TensorStream.variable_scope.get_variable(
+          TensorStream::GraphKeys::GLOBAL_STEP, shape: [],
+                                                dtype: :int64,
+                                                initializer: TensorStream.zeros_initializer,
+                                                trainable: false,
+                                                collections: [TensorStream::GraphKeys::GLOBAL_VARIABLES,
+                                                              TensorStream::GraphKeys::GLOBAL_STEP])
+      end
+
+      def get_global_step(graph = nil)
+        target_graph = graph || TensorStream.get_default_graph
+        global_step_tensors = target_graph.get_collection(TensorStream::GraphKeys::GLOBAL_STEP)
+        global_step_tensor = if global_step_tensors.nil? || global_step_tensors.empty?
+                                begin
+                                  target_graph.get_tensor_by_name('global_step:0')
+                                rescue TensorStream::KeyError
+                                  nil
+                                end
+                             elsif global_step_tensors.size == 1
+                               global_step_tensors[0]
+                             else
+                               TensorStream.logger.error("Multiple tensors in global_step collection.")
+                               nil
+                             end
+        global_step_tensor
+      end
+    end
+  end
+end

data/lib/tensor_stream/trainer.rb

@@ -3,6 +3,8 @@ require 'tensor_stream/train/saver'
 
 module TensorStream
   module Trainer
+    extend TensorStream::Train::Utils
+
     def self.write_graph(graph, path, filename, as_text: true, serializer: TensorStream::Pbtext)
       raise "only supports as_text=true for now" unless as_text
       new_filename = File.join(path, filename)

data/lib/tensor_stream/utils.rb

@@ -41,6 +41,9 @@ module TensorStream
       end.flatten
     end
 
+    ##
+    # Creates a variable
+    # A variable maintains state across sessions
     def variable(value, name: nil, initializer: nil, graph: nil, dtype: nil, trainable: true)
       op = Operation.new(:assign, nil, value)
       common_options = {
@@ -51,13 +54,13 @@ module TensorStream
         trainable: trainable
       }
       tensor = if value.is_a?(String)
-        TensorStream::Variable.new(dtype || :string, 0, [], common_options)
+        TensorStream::Variable.new(dtype || :string, 0, [], get_variable_scope, common_options)
       elsif value.is_a?(Integer)
-        TensorStream::Variable.new(dtype || :int32, 0, [], common_options)
+        TensorStream::Variable.new(dtype || :int32, 0, [], get_variable_scope, common_options)
       elsif value.is_a?(Float)
-        TensorStream::Variable.new(dtype || :float32, 0, [], common_options)
+        TensorStream::Variable.new(dtype || :float32, 0, [], get_variable_scope, common_options)
       else
-        TensorStream::Variable.new(dtype || :float32, 0, nil, common_options)
+        TensorStream::Variable.new(dtype || :float32, 0, nil, get_variable_scope, common_options)
       end
       op.inputs[0] = tensor
       tensor
@@ -65,16 +68,19 @@ module TensorStream
 
     def variable_scope(scope = nil, reuse: nil, initializer: nil)
       Thread.current[:tensor_stream_variable_scope] ||= []
-      Thread.current[:tensor_stream_variable_scope] << OpenStruct.new(name: scope, reuse: reuse, initializer: initializer)
+      variable_scope = VariableScope.new(name: scope, reuse: reuse, initializer: initializer)
+      Thread.current[:tensor_stream_variable_scope] << variable_scope
       scope_name = __v_scope_name
-      begin
-        if block_given?
+      if block_given?
+        begin
           TensorStream.get_default_graph.name_scope(scope) do
             yield(scope_name)
           end
+        ensure
+          Thread.current[:tensor_stream_variable_scope].pop
         end
-      ensure
-        Thread.current[:tensor_stream_variable_scope].pop
+      else
+        variable_scope
       end
     end
 
@@ -94,8 +100,8 @@ module TensorStream
     end
 
     def get_variable_scope
-      return nil unless Thread.current[:tensor_stream_variable_scope]
-      __v_scope_name
+      return VariableScope.new unless Thread.current[:tensor_stream_variable_scope]
+      Thread.current[:tensor_stream_variable_scope].last || VariableScope.new
     end
 
     def __v_scope_name
@@ -125,6 +131,8 @@ module TensorStream
         TensorStream::Tensor.new(dtype || :int32, 0, shape || [], shared_options)
       elsif value.is_a?(String)
         TensorStream::Tensor.new(dtype || :string, 0, shape || [], shared_options)
+      elsif !!value == value
+        TensorStream::Tensor.new(dtype || :boolean, 0, shape || [], shared_options)
       elsif value.is_a?(Array)
         dimension = shape || shape_eval(value)
         rank = dimension.size
@@ -146,7 +154,7 @@ module TensorStream
     end
 
     def get_variable(name, dtype: nil, shape: nil, initializer: nil, trainable: true, collections: nil)
-      TensorStream::Variable.new(dtype || :float32, nil, shape, collections: collections, name: name, initializer: initializer, trainable: trainable)
+      get_variable_scope.get_variable(name, dtype: dtype, shape: shape, initializer: initializer, trainable: trainable, collections: collections)
     end
 
     def get_collection(name, options = {})
@@ -158,6 +166,8 @@ module TensorStream
       ref.assign(value, name: name)
     end
 
+    ##
+    # Inserts a placeholder for a tensor that will be always fed.
     def placeholder(dtype, shape: nil, name: nil)
       TensorStream::Placeholder.new(dtype, nil, shape, name: name)
     end
@@ -208,9 +218,9 @@ module TensorStream
         input_a = convert_to_tensor(input_a)
         input_b = convert_to_tensor(input_b)
       end
-      
+
       if norm_dtype(input_a.data_type) != norm_dtype(input_b.data_type)
-        raise "Value Error: Tensor conversion requested dtype #{input_a.data_type} for tensor type #{input_b.data_type}" 
+        raise TensorStream::ValueError, "Value Error: Tensor conversion requested dtype #{input_a.data_type} for tensor type #{input_b.data_type}"
       end
 
       [input_a, input_b]

data/lib/tensor_stream/variable.rb

@@ -2,7 +2,7 @@ module TensorStream
   # Class that defines a TensorStream variable
   class Variable < Tensor
     attr_accessor :trainable, :options, :buffer
-    def initialize(data_type, rank, shape, options = {})
+    def initialize(data_type, rank, shape, variable_scope, options = {})
       setup_initial_state(options)
 
       @options = {
@@ -11,8 +11,10 @@ module TensorStream
       @rank = rank
       @value = nil
       @is_const = false
-      @name = [TensorStream.get_variable_scope, options[:name] || build_name].compact.reject(&:empty?).join('/')
-      @initalizer_tensor = options[:initializer] ? options[:initializer] : _variable_scope.initializer || TensorStream.glorot_uniform_initializer
+      scope_name = variable_scope ? variable_scope.name : nil
+      variable_scope_initializer = variable_scope ? variable_scope.initializer : nil
+      @name = [scope_name, options[:name] || build_name].compact.reject(&:empty?).join('/')
+      @initalizer_tensor = options[:initializer] ? options[:initializer] : variable_scope_initializer || TensorStream.glorot_uniform_initializer
       if shape.nil? && @initalizer_tensor && @initalizer_tensor.shape
         shape = @initalizer_tensor.shape.shape
       end
@@ -66,13 +68,5 @@ module TensorStream
     def self.global_variables_initializer
       variables_initializer(TensorStream::GraphKeys::GLOBAL_VARIABLES)
     end
-
-    private
-
-    def _variable_scope
-      return OpenStruct.new(name: '', reuse: false, initializer: nil) if Thread.current[:tensor_stream_variable_scope].nil? || Thread.current[:tensor_stream_variable_scope].empty?
-      scope = Thread.current[:tensor_stream_variable_scope].last
-      scope
-    end
   end
 end

data/lib/tensor_stream/variable_scope.rb

@@ -0,0 +1,15 @@
+module TensorStream
+  class VariableScope
+    attr_accessor :name, :reuse, :initializer
+
+    def initialize(name: '', reuse: nil, initializer: nil)
+      @name = name
+      @reuse = reuse
+      @initializer = initializer
+    end
+
+    def get_variable(name, dtype: nil, shape: nil, initializer: nil, trainable: true, collections: nil)
+      TensorStream::Variable.new(dtype || :float32, nil, shape, self, collections: collections, name: name, initializer: initializer, trainable: trainable)
+    end
+  end
+end

data/lib/tensor_stream/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
-  VERSION = '0.6.1'.freeze
+  VERSION = '0.7.0'.freeze
 
   def self.version
     VERSION