tensor_stream 0.8.0 → 0.8.1

data/lib/tensor_stream/ops.rb

@@ -55,7 +55,8 @@ module TensorStream
                          end
         tensor_program
       end
-      TensorStream.group(gs)
+
+      gs
     end
 
     ##

data/lib/tensor_stream/session.rb

@@ -71,7 +71,7 @@ module TensorStream
         value = delegate_to_evaluator(e, context, {})
         recursive_eval(value)
       end
-      result.size == 1 ? result.first : result
+      args.size == 1 ? result.first : result
     end
 
     def list_devices
@@ -111,17 +111,26 @@ module TensorStream
     end
 
     def delegate_to_evaluator(tensor_arr, session_context, context)
-      arr = tensor_arr.is_a?(Array) ? tensor_arr : [tensor_arr]
-      result = arr.collect do |tensor|
-        session_context[:_cache][:placement][tensor.name] = assign_evaluator(tensor) if session_context[:_cache][:placement][tensor.name].nil?
-
-        session_context[:_cache][:placement][tensor.name][1].run_with_buffer(tensor, session_context, context)
+      if tensor_arr.is_a?(Array)
+        tensor_arr.collect do |tensor|
+          if tensor.is_a?(Array)
+            delegate_to_evaluator(tensor, session_context, context)
+          else
+            run_with_session_context(tensor, session_context, context)
+          end
+        end
+      else
+        run_with_session_context(tensor_arr, session_context, context)
       end
-      result.size == 1 ? result.first : result
     end
 
     protected
 
+    def run_with_session_context(tensor, session_context, context)
+      session_context[:_cache][:placement][tensor.name] = assign_evaluator(tensor) if session_context[:_cache][:placement][tensor.name].nil?
+      session_context[:_cache][:placement][tensor.name][1].run_with_buffer(tensor, session_context, context)
+    end
+
     def recursive_eval(value, depth = 2)
       if value.is_a?(Array) && depth > 0
         value.collect { |v| recursive_eval(v, depth - 1) }
@@ -151,7 +160,7 @@ module TensorStream
     def prepare_evaluators(tensor_arr, context)
       context[:_cache][:placement] ||= {}
 
-      tensor_arr = tensor_arr.is_a?(Array) ? tensor_arr : [tensor_arr]
+      tensor_arr = tensor_arr.is_a?(Array) ? tensor_arr.flatten : [tensor_arr]
       tensor_arr.each do |tensor|
         next if context[:_cache][:placement][tensor.name]
 

data/lib/tensor_stream/train/adam_optimizer.rb

@@ -0,0 +1,87 @@
+module TensorStream
+  module Train
+    # High Level implementation of the gradient descent algorithm
+    class AdamOptimizer < Optimizer
+      include TensorStream::OpHelper
+
+      attr_accessor :learning_rate
+
+      def initialize(learning_rate = 0.001, beta1=0.9, beta2=0.999, epsilon = 1e-8,
+                     use_locking: false, name: "Adam")
+        @learning_rate = learning_rate
+        @beta1 = beta1
+        @beta2 = beta2
+        @epsilon = epsilon
+
+        # Tensor versions of the constructor arguments, created in _prepare().
+        @lr_t = nil
+        @beta1_t = nil
+        @beta2_t = nil
+        @epsilon_t = nil
+
+        # Created in SparseApply if needed.
+        @updated_lr = nil
+        super(name: name, use_locking: use_locking)
+      end
+
+      protected
+
+      def get_beta_accumulators
+        graph = TensorStream.get_default_graph
+        [ get_non_slot_variable("beta1_power", graph: graph),
+          get_non_slot_variable("beta2_power", graph: graph)]
+      end
+
+      def prepare
+        lr = call_if_callable(@learning_rate)
+        beta1 = call_if_callable(@beta1)
+        beta2 = call_if_callable(@beta2)
+        epsilon = call_if_callable(@epsilon)
+
+        @lr_t = TensorStream.convert_to_tensor(lr, name: "learning_rate")
+        @beta1_t = TensorStream.convert_to_tensor(beta1, name: "beta1")
+        @beta2_t = TensorStream.convert_to_tensor(beta2, name: "beta2")
+        @epsilon_t = TensorStream.convert_to_tensor(epsilon, name: "epsilon")
+      end
+
+      def create_slots(var_list)
+        first_var = var_list.min_by(&:name)
+        create_non_slot_variable(@beta1, "beta1_power", first_var)
+        create_non_slot_variable(@beta2, "beta2_power", first_var)
+
+        # Create slots for the first and second moments.
+        var_list.each do |v|
+          zeros_slot(v, "m", @name)
+          zeros_slot(v, "v", @name)
+        end
+      end
+
+      def apply_dense(grad, var)
+        m = get_slot(var, "m")
+        v = get_slot(var, "v")
+        beta1_power, beta2_power = get_beta_accumulators
+        _op(:apply_adam,
+            var, m, v,
+            TensorStream.cast(beta1_power, var.data_type),
+            TensorStream.cast(beta2_power, var.data_type),
+            TensorStream.cast(@lr_t, var.data_type),
+            TensorStream.cast(@beta1_t, var.data_type),
+            TensorStream.cast(@beta2_t, var.data_type),
+            TensorStream.cast(@epsilon_t, var.data_type),
+            grad, use_locking: @use_locking)
+      end
+
+      def finish(update_ops, name_scope)
+        TensorStream.control_dependencies(update_ops) do
+          beta1_power, beta2_power = get_beta_accumulators
+          update_beta1 = nil, update_beta2 = nil
+          TensorStream.colocate_with(beta1_power) do
+            update_beta1 = beta1_power.assign(beta1_power * @beta1_t, use_locking: @use_locking)
+            update_beta2 = beta2_power.assign(beta2_power * @beta2_t, use_locking: @use_locking)
+          end
+          TensorStream.group(update_ops + [update_beta1, update_beta2], name: name_scope)
+        end
+      end
+    end
+  end
+end

data/lib/tensor_stream/train/gradient_descent_optimizer.rb

@@ -6,8 +6,9 @@ module TensorStream
 
       attr_accessor :learning_rate
 
-      def initialize(learning_rate, _options = {})
+      def initialize(learning_rate, use_locking: false, name: "GradientDescent")
         @learning_rate = learning_rate
+        super(name: name, use_locking: use_locking)
       end
 
       protected

data/lib/tensor_stream/train/optimizer.rb

@@ -12,6 +12,7 @@ module TensorStream
         @use_locking = use_locking
         raise TensorStream::ValueError, "Must specify the optimizer name" unless @name
         @slots = {}
+        @non_slots = {}
       end
 
       def minimize(loss, var_list: nil, grad_loss: nil, global_step: nil, name: nil)
@@ -86,7 +87,7 @@ module TensorStream
         # no implementation
       end
 
-      def apply_dense(grad, var)
+      def apply_dense(_grad, _var)
         raise TensorStream::NotImplementedError, "not implemented"
       end
 
@@ -95,7 +96,7 @@ module TensorStream
       #
       # Args:
       #   var: Variable - A Variable object
-      #   slot_name: string - Name fot the slot
+      #   slot_name: string - Name for the slot
       #   op_name: string - Name to use when scoping the Variable that needs to be created
       def zeros_slot(var, slot_name, op_name)
         named_slots = slot_dict(slot_name)
@@ -124,6 +125,28 @@ module TensorStream
       def var_key(var)
         [var.op.graph, var.op.name]
       end
+
+      def get_non_slot_variable(name, graph: nil)
+        non_slot = @non_slots.fetch([name, graph], nil)
+        non_slot
+      end
+
+      def call_if_callable(param)
+        param.kind_of?(Proc) ? param.call : param
+      end
+
+
+      def create_non_slot_variable(initial_value, name, colocate_with)
+        graph = colocate_with.graph
+
+        key = [name, graph]
+        v = @non_slots.fetch(key, nil)
+        if v.nil?
+          v = TensorStream.variable(initial_value, name: name, trainable: false)
+          @non_slots[key] = v
+        end
+        v
+      end
     end
   end
 end

data/lib/tensor_stream/train/slot_creator.rb

@@ -23,7 +23,7 @@ module TensorStream
       #
       # Returns: A `Variable` object
       def create_slot(primary, val, name, colocate_with_primary: true)
-        TensorStream.variable_scope(primary.op.name + "/" + name) do
+        TensorStream.variable_scope(nil, primary.op.name + "/" + name) do
           if colocate_with_primary
             TensorStream.colocate_with(primary) do
               return create_slot_var(primary, val, "")

data/lib/tensor_stream/trainer.rb

@@ -2,6 +2,7 @@ require 'tensor_stream/train/slot_creator'
 require 'tensor_stream/train/optimizer'
 require 'tensor_stream/train/gradient_descent_optimizer'
 require 'tensor_stream/train/momentum_optimizer'
+require 'tensor_stream/train/adam_optimizer'
 require 'tensor_stream/train/saver'
 
 module TensorStream

data/lib/tensor_stream/utils.rb

@@ -66,9 +66,25 @@ module TensorStream
       tensor
     end
 
-    def variable_scope(scope = nil, reuse: nil, initializer: nil)
-      Thread.current[:tensor_stream_variable_scope] ||= []
+    ##
+    # Defines a variable context manager
+    def variable_scope(scope = nil, default_name = nil, reuse: nil, initializer: nil)
+      Thread.current[:tensor_stream_variable_scope] ||= [ VariableScope.new ]
+
+      # uniquenifier
+      if scope.nil? && default_name
+        same_names = get_variable_scope.used_names.select { |s| s.start_with?(default_name) }
+        new_name = default_name
+        index = 1
+        while same_names.include?(new_name)
+          new_name = "#{default_name}_#{index}"
+          index += 1
+        end
+        scope = new_name
+      end
+
       variable_scope = VariableScope.new(name: scope, reuse: reuse, initializer: initializer)
+      get_variable_scope.register_name(scope || "")
       Thread.current[:tensor_stream_variable_scope] << variable_scope
       scope_name = __v_scope_name
       if block_given?
@@ -100,8 +116,13 @@ module TensorStream
     end
 
     def get_variable_scope
-      return VariableScope.new unless Thread.current[:tensor_stream_variable_scope]
-      Thread.current[:tensor_stream_variable_scope].last || VariableScope.new
+      if !Thread.current[:tensor_stream_variable_scope]
+        variable_scope = VariableScope.new
+        Thread.current[:tensor_stream_variable_scope] = [variable_scope]
+        return variable_scope
+      end
+
+      Thread.current[:tensor_stream_variable_scope].last
     end
 
     def __v_scope_name

data/lib/tensor_stream/variable.rb

@@ -40,7 +40,7 @@ module TensorStream
       init_op
     end
 
-    def assign(value, name: nil)
+    def assign(value, name: nil, use_locking: false)
       _a, value = TensorStream.check_data_types(self, value)
       _op(:assign, self, value, name: name)
     end

data/lib/tensor_stream/variable_scope.rb

@@ -1,15 +1,21 @@
 module TensorStream
   class VariableScope
     attr_accessor :name, :reuse, :initializer
+    attr_reader :used_names
 
-    def initialize(name: '', reuse: nil, initializer: nil)
+    def initialize(name: nil, reuse: nil, initializer: nil)
       @name = name
       @reuse = reuse
       @initializer = initializer
+      @used_names = []
     end
 
     def get_variable(name, dtype: nil, shape: nil, initializer: nil, trainable: true, collections: nil, validate_shape: false)
       TensorStream::Variable.new(dtype || :float32, nil, shape, self, collections: collections, name: name, initializer: initializer, trainable: trainable)
     end
+
+    def register_name(name)
+      @used_names << name unless @used_names.include?(name)
+    end
   end
 end

data/lib/tensor_stream/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
-  VERSION = '0.8.0'.freeze
+  VERSION = '0.8.1'.freeze
 
   def self.version
     VERSION

data/samples/iris.rb

@@ -1,6 +1,5 @@
-require "bundler/setup"
 require 'tensor_stream'
-# require 'tensor_stream/evaluator/opencl/opencl_evaluator'
+require 'tensor_stream/evaluator/opencl/opencl_evaluator'
 
 # This neural network will predict the species of an iris based on sepal and petal size
 # Dataset: http://en.wikipedia.org/wiki/Iris_flower_data_set
@@ -66,8 +65,8 @@ end
 x_size = x_train[0].size
 y_size = y_train[0].size
 h_size = 256
-X = tf.placeholder(:float, shape: [nil, x_size])
-y = tf.placeholder(:float, shape: [nil, y_size])
+X = tf.placeholder(:float32, shape: [nil, x_size])
+y = tf.placeholder(:float32, shape: [nil, y_size])
 
 # Weight initializations
 w_1 = init_weights([x_size, h_size])
@@ -79,13 +78,17 @@ predict = tf.argmax(yhat, 1)
 
 # Backward propagation
 cost    = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels: y, logits: yhat))
-updates =  TensorStream::Train::MomentumOptimizer.new(0.01, 0.1, use_nesterov: true).minimize(cost)
+
+updates =  TensorStream::Train::GradientDescentOptimizer.new(0.01).minimize(cost)
+# updates =  TensorStream::Train::MomentumOptimizer.new(0.01, 0.5, use_nesterov: true).minimize(cost)
+# updates =  TensorStream::Train::AdamOptimizer.new.minimize(cost)
 
 # Run SGD
 sess = tf.session
 init = tf.global_variables_initializer
 sess.run(init)
-
+loss = sess.run(cost, feed_dict: { X => x_test, y => y_test })
+puts "loss test data set #{loss}"
 loss = sess.run(cost, feed_dict: { X => x_train, y => y_train })
 puts "Testing the untrained network..."
 puts loss

data/samples/linear_regression.rb

@@ -1,8 +1,8 @@
-require "bundler/setup"
+# Linear Regression sample, using SGD and auto-differentiation
+
 require 'tensor_stream'
-require 'benchmark'
 
-tf = TensorStream
+tf = TensorStream # use tf to make it look like TensorFlow
 
 learning_rate = 0.01
 momentum = 0.5
@@ -29,7 +29,9 @@ pred = X * W + b
 # Mean squared error
 cost = ((pred - Y) ** 2).reduce(:+) / ( 2 * n_samples)
 
-# optimizer = TensorStream::Train::MomentumOptimizer.new(learning_rate, momentum).minimize(cost)
+# Other possible Optimizers
+# optimizer = TensorStream::Train::MomentumOptimizer.new(learning_rate, momentum, use_nesterov: true).minimize(cost)
+# optimizer = TensorStream::Train::AdamOptimizer.new(learning_rate).minimize(cost)
 optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate).minimize(cost)
 
 # Initialize the variables (i.e. assign their default value)

data/samples/nearest_neighbor.rb

@@ -6,9 +6,9 @@ This example is using the MNIST database of handwritten digits
 Author: Aymeric Damien
 Project: https://github.com/aymericdamien/TensorFlow-Examples/
 
-Make sure to install the mnist-learn gem
+Make sure to install the mnist-learn gem !!
 '''
-require "bundler/setup"
+
 require 'tensor_stream'
 require 'mnist-learn'
 require 'tensor_stream/evaluator/opencl/opencl_evaluator'

data/{test_samples → samples}/raw_neural_net_sample.rb

@@ -9,12 +9,12 @@ Links:
 
 Author: Aymeric Damien
 Project: https://github.com/aymericdamien/TensorFlow-Examples/
+
+The mnist-learn gem is required as well as an OpenCL compatible device with drivers correctly installed
 """
-require "bundler/setup"
 require 'tensor_stream'
 require 'mnist-learn'
 require 'tensor_stream/evaluator/opencl/opencl_evaluator'
-require 'pry-byebug'
 
 tf = TensorStream
 # Import MNIST data
@@ -23,8 +23,9 @@ mnist = Mnist.read_data_sets('/tmp/data', one_hot: true)
 puts "downloading finished"
 
 # Parameters
-learning_rate = 0.1
-num_steps = 500
+learning_rate = 0.001
+momentum = 0.01
+num_steps = 100
 batch_size = 128
 display_step = 5
 
@@ -71,7 +72,7 @@ prediction = tf.nn.softmax(logits)
 loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(
     logits: logits, labels: Y))
 
-optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate)
+optimizer = TensorStream::Train::MomentumOptimizer.new(learning_rate, momentum, use_nesterov: true)
 train_op = optimizer.minimize(loss_op)
 
 # Evaluate model
@@ -88,28 +89,24 @@ tf.session do |sess|
     # Run the initializer
     sess.run(init)
 
-    # print("Testing Accuracy:", \
-    #     sess.run(accuracy, feed_dict: { X => mnist.test.images,
-    #                                     Y => mnist.test.labels}))
+    print("Testing Accuracy:", \
+        sess.run(accuracy, feed_dict: { X => mnist.test.images,
+                                        Y => mnist.test.labels}))
 
     (1..num_steps+1).each do |step|
         batch_x, batch_y = mnist.train.next_batch(batch_size)
         # Run optimization op (backprop)
-        puts "."
         sess.run(train_op, feed_dict: { X => batch_x, Y => batch_y })
         if step % display_step == 0 || step == 1
-        # Calculate batch loss and accuracy
-        loss, acc = sess.run([loss_op, accuracy], feed_dict: { X => batch_x, Y => batch_y})
-        print("Step " + step.to_s + ", Minibatch Loss= " + \
-                loss.to_s + ", Training Accuracy= " + \
-                acc.to_s)
+            # Calculate batch loss and accuracy
+            loss, acc = sess.run([loss_op, accuracy], feed_dict: { X => batch_x, Y => batch_y})
+            print("\nStep " + step.to_s + ", Minibatch Loss= " + \
+                    loss.to_s + ", Training Accuracy= " + \
+                    acc.to_s)
         end
     end
-
-    print("Optimization Finished!")
-
-    # Calculate accuracy for MNIST test images
-    print("Testing Accuracy:", \
-        sess.run(accuracy, feed_dict: {  X => mnist.test.images,
+    print("\nOptimization Finished!")
+    print("\nTesting Accuracy after optimization:", \
+        sess.run(accuracy, feed_dict: { X => mnist.test.images,
                                         Y => mnist.test.labels}))
 end