tensor_stream 0.8.5 → 0.8.6

data/lib/tensor_stream/session.rb

@@ -68,6 +68,7 @@ module TensorStream
         end
       end
       result = args.collect do |e|
+        next e.value if e.is_a?(Tensor) && e.is_const && e.value
         value = delegate_to_evaluator(e, context, {})
         recursive_eval(value)
       end

data/lib/tensor_stream/tensor.rb

@@ -27,9 +27,7 @@ module TensorStream
           # check if single dimenstion array is passed
           options[:value] = reshape(options[:value], shape.reverse.dup) if shape.size >= 2 && !options[:value].empty? && !options[:value][0].is_a?(Array)
 
-          @value = options[:value].collect do |v|
-            v.is_a?(Tensor) ? Tensor.cast_dtype(v, @data_type) : v
-          end
+          @value = options[:value].map { |v| v.is_a?(Tensor) ? Tensor.cast_dtype(v, @data_type) : v }
         elsif !shape.empty?
           @value = reshape(Tensor.cast_dtype(options[:value], @data_type), shape.dup)
         else
@@ -88,8 +86,7 @@ module TensorStream
     end
 
     def %(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:mod, self, TensorStream.convert_to_tensor(other, dtype: data_type))
+      TensorStream.mod(self, other)
     end
 
     def floor
@@ -292,8 +289,8 @@ module TensorStream
     end
 
     def add_consumer(consumer)
-      @consumers ||= []
-      @consumers << consumer.name if !@consumers.include?(consumer.name) && consumer.name != name
+      @consumers ||= Set.new
+      @consumers << consumer.name if consumer.name != name
     end
 
     def setup_output(consumer)

data/lib/tensor_stream/tensor_shape.rb

@@ -23,6 +23,10 @@ module TensorStream
       shape.size
     end
 
+    def scalar?
+      shape.size.zero?
+    end
+
     def known?
       return false if shape.nil?
       shape.each { |s| return false if s.nil? }
@@ -54,8 +58,13 @@ module TensorStream
     end
 
     def self.reshape(arr, new_shape)
-      return arr if new_shape.empty?
+      arr = arr.is_a?(Array) ? arr.flatten : [arr]
+
+      new_shape = TensorShape.fix_inferred_elements(new_shape, arr.size)
+      return arr[0] if arr.size == 1 && new_shape.empty?
+
       new_shape = new_shape.dup
+
       s = new_shape.shift
 
       if new_shape.size.zero?

data/lib/tensor_stream/train/adagrad_optimizer.rb

@@ -0,0 +1,46 @@
+module TensorStream
+  module Train
+    # High Level implementation of the Adagrad algorithm
+    class AdagradOptimizer < Optimizer
+      include TensorStream::OpHelper
+
+      attr_accessor :learning_rate
+
+      def initialize(learning_rate, initial_accumulator_value = 0.1,
+        use_locking: false, name: "Adagrad")
+        @learning_rate = learning_rate
+        @initial_accumulator_value = initial_accumulator_value
+        @learning_rate_tensor = nil
+        super(name: name, use_locking: use_locking)
+      end
+
+      protected
+
+      def create_slots(var_list)
+        var_list.each do |v|
+          dtype = v.data_type
+          init = nil
+          if v.shape.known?
+            init = TensorStream.constant_initializer(@initial_accumulator_value, dtype: dtype)
+          else
+            init_constant = TensorStream.fill(TensorStream.shape(v), @initial_accumulator_value)
+            init = TensorStream.cast(init_constant, dtype)
+          end
+          get_or_make_slot_with_initializer(v, init, v.shape, dtype, "accumulator", @name)
+        end
+      end
+
+      def prepare
+        learning_rate = call_if_callable(@learning_rate)
+        @learning_rate_tensor = TensorStream.convert_to_tensor(learning_rate, name: "learning_rate")
+      end
+
+      def apply_dense(grad, var)
+        acc = get_slot(var, "accumulator")
+        _op(:apply_adagrad,
+            var, acc, TensorStream.cast(@learning_rate_tensor, var.data_type),
+            grad, use_locking: @use_locking)
+      end
+    end
+  end
+end

data/lib/tensor_stream/train/optimizer.rb

@@ -147,6 +147,18 @@ module TensorStream
         end
         v
       end
+
+      ##
+      # Find or create a slot for a variable, using an Initializer.
+      def get_or_make_slot_with_initializer(var, initializer, shape, dtype, slot_name, op_name)
+        named_slots = slot_dict(slot_name)
+        if !named_slots.key?(var_key(var))
+          new_slot_variable = create_slot_with_initializer(
+            var, initializer, shape, dtype, op_name)
+          named_slots[var_key(var)] = new_slot_variable
+        end
+        named_slots[var_key(var)]
+      end
     end
   end
 end

data/lib/tensor_stream/train/rmsprop_optimizer.rb

@@ -0,0 +1,84 @@
+module TensorStream
+  module Train
+    # High Level implementation of the RMSProp algorithm
+    # This is a straight port from TensorFlows rmsprop.py
+    class RMSPropOptimizer < Optimizer
+      include TensorStream::OpHelper
+
+      attr_accessor :learning_rate
+
+      ##
+      # Optimizer that implements the RMSProp algorithm.
+      #
+      # [paper](http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf).
+      def initialize(learning_rate, decay = 0.9, momentum = 0.0, epsilon = 1e-10, centered: false,
+                     use_locking: false, name: "RMSProp")
+        @learning_rate = learning_rate
+        @decay = decay
+        @momentum = momentum
+        @epsilon = epsilon
+        @centered = centered
+
+        # Tensor versions of the constructor arguments, created in _prepare().
+        @learning_rate_tensor = nil
+        @decay_tensor = nil
+        @momentum_tensor = nil
+        @epsilon_tensor = nil
+
+        super(name: name, use_locking: use_locking)
+      end
+
+      protected
+
+      def prepare
+        lr = call_if_callable(@learning_rate)
+        decay = call_if_callable(@decay)
+        momentum = call_if_callable(@momentum)
+        epsilon = call_if_callable(@epsilon)
+
+        @learning_rate_tensor = TensorStream.convert_to_tensor(lr, name: "learning_rate")
+        @decay_tensor = TensorStream.convert_to_tensor(decay, name: "decay")
+        @momentum_tensor = TensorStream.convert_to_tensor(momentum, name: "momentum")
+        @epsilon_tensor = TensorStream.convert_to_tensor(epsilon, name: "epsilon")
+      end
+
+      def create_slots(var_list)
+        # Create slots for the first and second moments.
+        var_list.each do |v|
+          init_rms = if v.shape.known?
+                       TensorStream.ones_initializer(dtype: v.data_type)
+                     else
+                       TensorStream.ones_like(v)
+                     end
+
+          get_or_make_slot_with_initializer(v, init_rms, v.shape, v.data_type, "rms", @name)
+
+          zeros_slot(v, "mg", @name) if @centered
+          zeros_slot(v, "momentum", @name)
+        end
+      end
+
+      def apply_dense(grad, var)
+        rms = get_slot(var, "rms")
+        mom = get_slot(var, "momentum")
+
+        if @centered
+          mg = get_slot(var, "mg")
+          _op(:apply_centered_rms_prop, var, mg, rms, mom,
+              TensorStream.cast(@learning_rate_tensor, var.data_type),
+              TensorStream.cast(@decay_tensor, var.data_type),
+              TensorStream.cast(@momentum_tensor, var.data_type),
+              TensorStream.cast(@epsilon_tensor, var.data_type),
+              grad, use_locking: @use_locking)
+        else
+          _op(:apply_rms_prop, var, rms, mom,
+              TensorStream.cast(@learning_rate_tensor, var.data_type),
+              TensorStream.cast(@decay_tensor, var.data_type),
+              TensorStream.cast(@momentum_tensor, var.data_type),
+              TensorStream.cast(@epsilon_tensor, var.data_type),
+              grad, use_locking: @use_locking)
+        end
+      end
+    end
+  end
+end

data/lib/tensor_stream/train/slot_creator.rb

@@ -5,9 +5,9 @@ module TensorStream
 
       ##
       # Helper function for creating a slot variable.
-      def create_slot_var(primary, val, scope)
-        slot = get_variable(scope, initializer: val, trainable: false,
-                                                  validate_shape: val.shape.is_fully_defined?)
+      def create_slot_var(primary, val, scope, shape)
+        slot = get_variable(scope, initializer: val, trainable: false, shape: shape,
+                                                  validate_shape: val.shape && val.shape.known?)
         slot
       end
 
@@ -24,16 +24,21 @@ module TensorStream
       # Returns: A `Variable` object
       def create_slot(primary, val, name, colocate_with_primary: true)
         TensorStream.variable_scope(nil, primary.op.name + "/" + name) do
-          if colocate_with_primary
-            TensorStream.colocate_with(primary) do
-              return create_slot_var(primary, val, "")
-            end
-          else
-            return create_slot_var(primary, val, "")
+          return create_slot_var(primary, val, "", nil) if colocate_with_primary
+
+          TensorStream.colocate_with(primary) do
+            return create_slot_var(primary, val, "", nil)
           end
         end
       end
 
+      def create_slot_with_initializer(primary, initializer, shape, dtype, name, colocate_with_primary: true)
+        prefix = primary.op.name
+        TensorStream.variable_scope(nil, prefix + "/" + name) do
+          create_slot_var(primary, initializer, "", shape.shape)
+        end
+      end
+
       ##
       # Create a slot initialized to 0 with same shape as the primary object.
       #

data/lib/tensor_stream/trainer.rb

@@ -4,6 +4,8 @@ require 'tensor_stream/train/gradient_descent_optimizer'
 require 'tensor_stream/train/momentum_optimizer'
 require 'tensor_stream/train/adam_optimizer'
 require 'tensor_stream/train/adadelta_optimizer'
+require 'tensor_stream/train/adagrad_optimizer'
+require 'tensor_stream/train/rmsprop_optimizer'
 require 'tensor_stream/train/saver'
 
 module TensorStream

data/lib/tensor_stream/utils.rb

@@ -225,13 +225,15 @@ module TensorStream
     end
 
     def convert_to_tensor(value, dtype: nil, name: nil, preferred_dtype: nil)
+      return value if value.is_a?(Tensor)
       return convert_to_tensor(value.call) if value.is_a?(Proc)
+      if value.is_a?(Array) && value[0].is_a?(Tensor)
+        return TensorStream.stack(value) if value.size > 1
 
-      if !value.is_a?(Tensor)
-        i_cons(value, dtype: dtype || Tensor.detect_type(value), name: name)
-      else
-        value
+        return TensorStream.expand_dims(value[0], 0)
       end
+
+      i_cons(value, dtype: dtype || Tensor.detect_type(value), name: name)
     end
 
     def check_allowed_types(input, types)

data/lib/tensor_stream/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
-  VERSION = '0.8.5'.freeze
+  VERSION = '0.8.6'.freeze
 
   def self.version
     VERSION

data/samples/iris.rb

@@ -1,5 +1,6 @@
+require "bundler/setup"
 require 'tensor_stream'
-require 'tensor_stream/evaluator/opencl/opencl_evaluator'
+# require 'tensor_stream/evaluator/opencl/opencl_evaluator'
 
 # This neural network will predict the species of an iris based on sepal and petal size
 # Dataset: http://en.wikipedia.org/wiki/Iris_flower_data_set
@@ -79,9 +80,9 @@ predict = tf.argmax(yhat, 1)
 # Backward propagation
 cost    = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels: y, logits: yhat))
 
-updates =  TensorStream::Train::GradientDescentOptimizer.new(0.01).minimize(cost)
+# updates =  TensorStream::Train::GradientDescentOptimizer.new(0.01).minimize(cost)
 # updates =  TensorStream::Train::MomentumOptimizer.new(0.01, 0.5, use_nesterov: true).minimize(cost)
-# updates =  TensorStream::Train::AdamOptimizer.new.minimize(cost)
+updates =  TensorStream::Train::RMSPropOptimizer.new(0.01).minimize(cost)
 
 # Run SGD
 sess = tf.session

data/samples/linear_regression.rb

@@ -31,9 +31,12 @@ pred = X * W + b
 cost = ((pred - Y) ** 2).reduce(:+) / ( 2 * n_samples)
 
 # Other possible Optimizers
+
 # optimizer = TensorStream::Train::MomentumOptimizer.new(learning_rate, momentum, use_nesterov: true).minimize(cost)
 # optimizer = TensorStream::Train::AdamOptimizer.new(learning_rate).minimize(cost)
 # optimizer = TensorStream::Train::AdadeltaOptimizer.new(1.0).minimize(cost)
+# optimizer = TensorStream::Train::AdagradOptimizer.new(0.01).minimize(cost)
+# optimizer = TensorStream::Train::RMSPropOptimizer.new(0.01, centered: true).minimize(cost)
 optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate).minimize(cost)
 
 

data/samples/rnn.rb

@@ -0,0 +1,105 @@
+# RNN sample
+#
+# Ruby port Example based on article by Erik Hallström
+# https://medium.com/@erikhallstrm/hello-world-rnn-83cd7105b767
+#
+#
+
+require "bundler/setup"
+require 'tensor_stream'
+
+tf = TensorStream
+
+num_epochs = 100
+total_series_length = 50000
+truncated_backprop_length = 15
+state_size = 4
+num_classes = 2
+echo_step = 3
+batch_size = 5
+num_batches = total_series_length / batch_size / truncated_backprop_length
+randomizer = TensorStream.random_uniform([total_series_length], minval: 0, maxval: 2)
+
+
+def generate_data(randomizer, total_series_length, batch_size, echo_step)
+  x = randomizer.eval
+  y = x.rotate(-echo_step)
+
+  y[echo_step] = 0
+
+  x = TensorStream::TensorShape.reshape(x, [batch_size, -1])  # The first index changing slowest, subseries as rows
+  y = TensorStream::TensorShape.reshape(y, [batch_size, -1])
+  [x, y]
+end
+
+batchX_placeholder = tf.placeholder(:float32, shape: [batch_size, truncated_backprop_length], name: 'batch_x')
+batchY_placeholder = tf.placeholder(:int32, shape: [batch_size, truncated_backprop_length], name: 'batch_y')
+
+init_state = tf.placeholder(:float32, shape: [batch_size, state_size], name: 'init_state')
+
+
+W = tf.variable(tf.random_uniform([state_size+1, state_size]), dtype: :float32, name: 'W')
+b = tf.variable(tf.zeros([state_size]), dtype: :float32, name: 'b')
+
+W2 = tf.variable(tf.random_uniform([state_size, num_classes]), dtype: :float32, name: 'W2')
+b2 = tf.variable(tf.zeros([num_classes]), dtype: :float32, name: 'b2')
+
+
+inputs_series = tf.unpack(batchX_placeholder, axis: 1)
+labels_series = tf.unpack(batchY_placeholder, axis: 1)
+
+current_state = init_state
+states_series = []
+
+inputs_series.each do |current_input|
+  current_input = tf.reshape(current_input, [batch_size, 1])
+  input_and_state_concatenated = tf.concat([current_input, current_state], 1)  # Increasing number of columns
+  next_state = tf.tanh(tf.matmul(input_and_state_concatenated, W) + b)  # Broadcasted addition
+  states_series << next_state
+  current_state = next_state
+end
+
+logits_series = states_series.collect do |state|
+  tf.matmul(state, W2) + b2
+end
+
+predictions_series = logits_series.collect do |logits|
+  tf.nn.softmax(logits)
+end
+
+losses = logits_series.zip(labels_series).collect do |logits, labels|
+  tf.nn.sparse_softmax_cross_entropy_with_logits(logits: logits, labels: labels)
+end
+total_loss = tf.reduce_mean(losses)
+
+train_step = TensorStream::Train::AdagradOptimizer.new(0.3).minimize(total_loss)
+
+puts "#{tf.get_default_graph.nodes.keys.size} nodes created"
+zeros_state = tf.zeros([batch_size, state_size]).eval
+tf.session do |sess|
+  sess.run(tf.global_variables_initializer)
+  (0..num_epochs).each do |epoch_idx|
+    x, y = generate_data(randomizer, total_series_length, batch_size, echo_step)
+    _current_state = zeros_state
+    print("New data, epoch", epoch_idx, "\n")
+    (0..num_batches - 1).each do |batch_idx|
+      start_idx = batch_idx * truncated_backprop_length
+      end_idx = start_idx + truncated_backprop_length
+
+      batchX = x.map { |x| x[start_idx...end_idx] }
+      batchY = y.map { |y| y[start_idx...end_idx] }
+
+      _total_loss, _train_step, _current_state, _predictions_series = sess.run(
+          [total_loss, train_step, current_state, predictions_series],
+          feed_dict: {
+              batchX_placeholder => batchX,
+              batchY_placeholder => batchY,
+              init_state => _current_state
+          })
+
+      if batch_idx%100 == 0
+          print("Step",batch_idx, " Loss ", _total_loss, "\n")
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tensor_stream
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.8.6
 platform: ruby
 authors:
 - Joseph Emmanuel Dayo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-06 00:00:00.000000000 Z
+date: 2018-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -261,6 +261,7 @@ files:
 - lib/tensor_stream/evaluator/operation_helpers/math_helper.rb
 - lib/tensor_stream/evaluator/operation_helpers/random_gaussian.rb
 - lib/tensor_stream/evaluator/ruby/array_ops.rb
+- lib/tensor_stream/evaluator/ruby/check_ops.rb
 - lib/tensor_stream/evaluator/ruby/images_ops.rb
 - lib/tensor_stream/evaluator/ruby/math_ops.rb
 - lib/tensor_stream/evaluator/ruby/nn_ops.rb
@@ -288,10 +289,12 @@ files:
 - lib/tensor_stream/tensor.rb
 - lib/tensor_stream/tensor_shape.rb
 - lib/tensor_stream/train/adadelta_optimizer.rb
+- lib/tensor_stream/train/adagrad_optimizer.rb
 - lib/tensor_stream/train/adam_optimizer.rb
 - lib/tensor_stream/train/gradient_descent_optimizer.rb
 - lib/tensor_stream/train/momentum_optimizer.rb
 - lib/tensor_stream/train/optimizer.rb
+- lib/tensor_stream/train/rmsprop_optimizer.rb
 - lib/tensor_stream/train/saver.rb
 - lib/tensor_stream/train/slot_creator.rb
 - lib/tensor_stream/train/utils.rb
@@ -311,6 +314,7 @@ files:
 - samples/multigpu.rb
 - samples/nearest_neighbor.rb
 - samples/raw_neural_net_sample.rb
+- samples/rnn.rb
 - tensor_stream.gemspec
 homepage: http://www.github.com/jedld/tensor_stream
 licenses: