tensor_stream 0.9.2 → 0.9.5

data/lib/tensor_stream/profile/report_tool.rb

@@ -5,13 +5,13 @@ module TensorStream
     def self.profile_for(session, order_by: :slowest)
       context = session.last_session_context
       eval_times = context[:profile][:operations].map do |name, profile|
-        [name, profile[:eval_time], profile[:tensor].source]
+        [name, profile[:eval_time], profile[:shape], profile[:tensor].source]
       end
 
       if order_by == :slowest
-        eval_times.sort_by { |a, b| b[1] <=> a[1] }
+        eval_times.sort_by { |a| a[1] }.reverse!
       else
-        eval_times.sort_by { |a, b| a[1] <=> b[1] }
+        eval_times.sort_by { |a| a[1] }
       end
     end
   end

data/lib/tensor_stream/tensor_shape.rb

@@ -20,7 +20,7 @@ module TensorStream
     end
 
     def ndims
-      shape.size
+      shape ? shape.size : nil
     end
 
     def scalar?
@@ -29,20 +29,21 @@ module TensorStream
 
     def known?
       return false if shape.nil?
-      
+
       a_shape = shape.is_a?(Array) ? shape : [shape]
       a_shape.each { |s| return false if s.nil? || s < 0 }
 
       true
     end
 
-    def is_fully_defined?
+    def fully_defined?
       known?
     end
 
     def self.infer_shape(shape_a, shape_b)
-      return shape_a if shape_b.nil?
-      return shape_b if shape_a.nil?
+      return nil if shape_a.nil? || shape_b.nil?
+      return shape_a if shape_b.empty?
+      return shape_b if shape_a.empty?
       return shape_a if shape_a == shape_b
       return shape_b if shape_b.size > shape_a.size
       return shape_a if shape_a.size > shape_b.size
@@ -55,13 +56,14 @@ module TensorStream
         next nil if s.nil? || reversed_b[index].nil?
         next nil if s.is_a?(Tensor) || reversed_b[index].is_a?(Tensor)
         next reversed_b[index] if reversed_b[index] > s
+
         s
       end.reverse
     end
 
     def self.reshape(arr, new_shape)
       arr = arr.is_a?(Array) ? arr.flatten : [arr]
-
+      new_shape = new_shape.is_a?(TensorShape) ? new_shape.shape : new_shape
       new_shape = TensorShape.fix_inferred_elements(new_shape, arr.size)
       return arr[0] if arr.size == 1 && new_shape.empty?
 
@@ -71,6 +73,7 @@ module TensorStream
 
       if new_shape.size.zero?
         raise "reshape dimen mismatch #{arr.size} != #{s}" if arr.size != s
+
         return arr
       end
 

data/lib/tensor_stream/train/adadelta_optimizer.rb

@@ -11,7 +11,7 @@ module TensorStream
         @learning_rate = learning_rate
         @rho = rho
         @epsilon = epsilon
-    
+
         # Tensor versions of the constructor arguments, created in _prepare().
         @learning_rate_tensor = nil
         @rho_t = nil

data/lib/tensor_stream/train/adagrad_optimizer.rb

@@ -7,7 +7,7 @@ module TensorStream
       attr_accessor :learning_rate
 
       def initialize(learning_rate, initial_accumulator_value = 0.1,
-        use_locking: false, name: "Adagrad")
+                     use_locking: false, name: "Adagrad")
         @learning_rate = learning_rate
         @initial_accumulator_value = initial_accumulator_value
         @learning_rate_tensor = nil

data/lib/tensor_stream/train/adam_optimizer.rb

@@ -43,8 +43,8 @@ module TensorStream
 
       def get_beta_accumulators
         graph = TensorStream.get_default_graph
-        [ get_non_slot_variable("beta1_power", graph: graph),
-          get_non_slot_variable("beta2_power", graph: graph)]
+        [get_non_slot_variable("beta1_power", graph: graph),
+         get_non_slot_variable("beta2_power", graph: graph)]
       end
 
       def prepare

data/lib/tensor_stream/train/learning_rate_decay.rb

@@ -0,0 +1,29 @@
+##
+# Ruby port of tensorflow's learning rate decay functions
+module TensorStream
+  module Train
+    module LearningRateDecay
+      include TensorStream::Utils
+      include TensorStream::OpHelper
+      include TensorStream::Ops
+
+      ##
+      # Applies exponential decay to the learning rate
+      def exponential_decay(learning_rate, global_step, decay_steps, decay_rate, staircase: false, name: nil)
+        raise TensorStream::ValueError, "global_step is required for exponential_decay." if global_step.nil?
+
+        name_scope(name, default: 'ExponentialDecay', values: [learning_rate, global_step, decay_steps, decay_rate]) do
+          learning_rate = convert_to_tensor(learning_rate, name: "learning_rate")
+          data_type = learning_rate.data_type
+          decay_steps = cast(decay_steps, data_type)
+          decay_rate = cast(decay_rate, data_type)
+
+          global_step_recomp = cast(global_step, data_type)
+          p = global_step_recomp / decay_steps
+          p = floor(p) if staircase
+          multiply(learning_rate, pow(decay_rate, p), name: name)
+        end
+      end
+    end
+  end
+end

data/lib/tensor_stream/train/optimizer.rb

@@ -11,6 +11,7 @@ module TensorStream
         @name = name
         @use_locking = use_locking
         raise TensorStream::ValueError, "Must specify the optimizer name" unless @name
+
         @slots = {}
         @non_slots = {}
       end
@@ -51,6 +52,7 @@ module TensorStream
       def compute_gradients(loss, var_list: nil, grad_loss: nil)
         trainable_vars = if var_list
                            raise "var_list must be an array" unless var_list.is_a?(Array)
+
                            var_list.each_with_index { |var, index| raise "var #{index} not a Variable" unless var.is_a?(Variable) }
 
                            var_list
@@ -66,6 +68,7 @@ module TensorStream
       def get_slot(var, name)
         named_slots = @slots.fetch(name, nil)
         return nil if named_slots.nil?
+
         named_slots.fetch(var_key(var), nil)
       end
 
@@ -100,7 +103,7 @@ module TensorStream
       #   op_name: string - Name to use when scoping the Variable that needs to be created
       def zeros_slot(var, slot_name, op_name)
         named_slots = slot_dict(slot_name)
-        if !named_slots.key?(var_key(var))
+        unless named_slots.key?(var_key(var))
           named_slots[var_key(var)] = create_zeros_slot(var, op_name)
         end
         named_slots[var_key(var)]
@@ -132,10 +135,9 @@ module TensorStream
       end
 
       def call_if_callable(param)
-        param.kind_of?(Proc) ? param.call : param
+        param.is_a?(Proc) ? param.call : param
       end
 
-
       def create_non_slot_variable(initial_value, name, colocate_with)
         graph = colocate_with.graph
 
@@ -152,9 +154,8 @@ module TensorStream
       # Find or create a slot for a variable, using an Initializer.
       def get_or_make_slot_with_initializer(var, initializer, shape, dtype, slot_name, op_name)
         named_slots = slot_dict(slot_name)
-        if !named_slots.key?(var_key(var))
-          new_slot_variable = create_slot_with_initializer(
-            var, initializer, shape, dtype, op_name)
+        unless named_slots.key?(var_key(var))
+          new_slot_variable = create_slot_with_initializer(var, initializer, shape, dtype, op_name)
           named_slots[var_key(var)] = new_slot_variable
         end
         named_slots[var_key(var)]

data/lib/tensor_stream/train/saver.rb

@@ -80,6 +80,7 @@ module TensorStream
 
       def _add_saveable(saveables, seen_ops, saveable)
         raise TensorStream::ValueError, "The same saveable will be restored with two names: #{saveable.name}" if seen_ops.include?(saveable.op)
+
         saveables << saveable
         seen_ops << saveable.op
       end

data/lib/tensor_stream/train/slot_creator.rb

@@ -5,7 +5,7 @@ module TensorStream
 
       ##
       # Helper function for creating a slot variable.
-      def create_slot_var(primary, val, scope, shape)
+      def create_slot_var(_primary, val, scope, shape)
         slot = get_variable(scope, initializer: val, trainable: false, shape: shape,
                                                   validate_shape: val.shape && val.shape.known?)
         slot
@@ -53,7 +53,7 @@ module TensorStream
       def create_zeros_slot(primary, name, dtype: nil, colocate_with_primary: true)
         dtype = primary.data_type if dtype.nil?
         slot_shape = primary.shape
-        slot_shape = if slot_shape.is_fully_defined?
+        slot_shape = if slot_shape.fully_defined?
                        slot_shape.shape
                      else
                        TensorStream.shape(primary.initialized_value)

data/lib/tensor_stream/trainer.rb

@@ -7,13 +7,16 @@ require 'tensor_stream/train/adadelta_optimizer'
 require 'tensor_stream/train/adagrad_optimizer'
 require 'tensor_stream/train/rmsprop_optimizer'
 require 'tensor_stream/train/saver'
+require 'tensor_stream/train/learning_rate_decay'
 
 module TensorStream
   module Trainer
     extend TensorStream::Train::Utils
+    extend TensorStream::Train::LearningRateDecay
 
     def self.write_graph(graph, path, filename, as_text: true, serializer: TensorStream::Pbtext)
       raise "only supports as_text=true for now" unless as_text
+
       new_filename = File.join(path, filename)
       File.write(new_filename, serializer.new.get_string(graph))
     end

data/lib/tensor_stream/utils.rb

@@ -104,13 +104,13 @@ module TensorStream
       get_default_graph.device(device_uri, &block)
     end
 
-    def name_scope(name, default: nil, values: nil)
+    def name_scope(name, default_name = nil, default: nil, values: nil)
       if values
         graph_count = values.select { |v| v.is_a?(Tensor) }.map(&:graph).map(&:object_id).uniq.size
         raise "values are not on the same graph" if graph_count > 1
       end
 
-      get_default_graph.name_scope(name || default) do |scope|
+      get_default_graph.name_scope(name || default_name || default) do |scope|
         yield scope if block_given?
       end
     end

data/lib/tensor_stream/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
-  VERSION = '0.9.2'.freeze
+  VERSION = '0.9.5'.freeze
 
   def self.version
     VERSION

data/lib/tensor_stream.rb

@@ -32,12 +32,16 @@ require "tensor_stream/debugging/debugging"
 require 'tensor_stream/utils'
 require 'tensor_stream/train/utils'
 require 'tensor_stream/images'
-require 'tensor_stream/trainer'
+
 require 'tensor_stream/profile/report_tool'
 
 # require 'tensor_stream/libraries/layers'
+require 'tensor_stream/monkey_patches/patch'
 require 'tensor_stream/monkey_patches/integer'
+require 'tensor_stream/monkey_patches/float'
+require 'tensor_stream/monkey_patches/array'
 require 'tensor_stream/ops'
+require 'tensor_stream/trainer'
 
 # module that exposes TensorStream top level functions
 module TensorStream

data/samples/rnn.rb

@@ -0,0 +1,108 @@
+# RNN sample
+#
+# Ruby port Example based on article by Erik Hallström
+# https://medium.com/@erikhallstrm/hello-world-rnn-83cd7105b767
+#
+#
+
+require "bundler/setup"
+require 'tensor_stream'
+# require 'tensor_stream/opencl'
+require 'pry-byebug'
+
+
+tf = TensorStream
+
+num_epochs = 100
+total_series_length = 50000
+truncated_backprop_length = 15
+state_size = 4
+num_classes = 2
+echo_step = 3
+batch_size = 5
+num_batches = total_series_length / batch_size / truncated_backprop_length
+randomizer = TensorStream.random_uniform([total_series_length], minval: 0, maxval: 2)
+
+
+def generate_data(randomizer, total_series_length, batch_size, echo_step)
+  x = randomizer.eval
+  y = x.rotate(-echo_step)
+
+  y[echo_step] = 0
+
+  x = TensorStream::TensorShape.reshape(x, [batch_size, -1])  # The first index changing slowest, subseries as rows
+  y = TensorStream::TensorShape.reshape(y, [batch_size, -1])
+  [x, y]
+end
+
+batchX_placeholder = tf.placeholder(:float32, shape: [batch_size, truncated_backprop_length], name: 'batch_x')
+batchY_placeholder = tf.placeholder(:int32, shape: [batch_size, truncated_backprop_length], name: 'batch_y')
+
+init_state = tf.placeholder(:float32, shape: [batch_size, state_size], name: 'init_state')
+
+
+W = tf.variable(tf.random_uniform([state_size+1, state_size]), dtype: :float32, name: 'W')
+b = tf.variable(tf.zeros([state_size]), dtype: :float32, name: 'b')
+
+W2 = tf.variable(tf.random_uniform([state_size, num_classes]), dtype: :float32, name: 'W2')
+b2 = tf.variable(tf.zeros([num_classes]), dtype: :float32, name: 'b2')
+
+
+inputs_series = tf.unpack(batchX_placeholder, axis: 1)
+labels_series = tf.unpack(batchY_placeholder, axis: 1)
+
+current_state = init_state
+states_series = []
+
+inputs_series.each do |current_input|
+  current_input = tf.reshape(current_input, [batch_size, 1])
+  input_and_state_concatenated = tf.concat([current_input, current_state], 1)  # Increasing number of columns
+  next_state = tf.tanh(tf.matmul(input_and_state_concatenated, W) + b)  # Broadcasted addition
+  states_series << next_state
+  current_state = next_state
+end
+
+logits_series = states_series.collect do |state|
+  tf.matmul(state, W2) + b2
+end
+
+predictions_series = logits_series.collect do |logits|
+  tf.nn.softmax(logits)
+end
+
+losses = logits_series.zip(labels_series).collect do |logits, labels|
+  tf.nn.sparse_softmax_cross_entropy_with_logits(logits: logits, labels: labels)
+end
+total_loss = tf.reduce_mean(losses)
+
+train_step = TensorStream::Train::AdagradOptimizer.new(0.1).minimize(total_loss)
+
+puts "#{tf.get_default_graph.nodes.keys.size} nodes created"
+zeros_state = tf.zeros([batch_size, state_size]).eval
+tf.session do |sess|
+  sess.run(tf.global_variables_initializer)
+  (0..num_epochs).each do |epoch_idx|
+    x, y = generate_data(randomizer, total_series_length, batch_size, echo_step)
+    _current_state = zeros_state
+    print("New data, epoch", epoch_idx, "\n")
+    (0..num_batches - 1).each do |batch_idx|
+      start_idx = batch_idx * truncated_backprop_length
+      end_idx = start_idx + truncated_backprop_length
+
+      batchX = x.map { |x| x[start_idx...end_idx] }
+      batchY = y.map { |y| y[start_idx...end_idx] }
+
+      _total_loss, _train_step, _current_state, _predictions_series = sess.run(
+          [total_loss, train_step, current_state, predictions_series],
+          feed_dict: {
+              batchX_placeholder => batchX,
+              batchY_placeholder => batchY,
+              init_state => _current_state
+          })
+
+      if batch_idx%10 == 0
+          print("Step",batch_idx, " Loss ", _total_loss, "\n")
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tensor_stream
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.5
 platform: ruby
 authors:
 - Joseph Emmanuel Dayo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-21 00:00:00.000000000 Z
+date: 2018-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -275,12 +275,16 @@ files:
 - lib/tensor_stream/graph_serializers/packer.rb
 - lib/tensor_stream/graph_serializers/pbtext.rb
 - lib/tensor_stream/graph_serializers/serializer.rb
+- lib/tensor_stream/helpers/infer_shape.rb
 - lib/tensor_stream/helpers/op_helper.rb
 - lib/tensor_stream/helpers/string_helper.rb
 - lib/tensor_stream/images.rb
 - lib/tensor_stream/initializer.rb
 - lib/tensor_stream/math_gradients.rb
+- lib/tensor_stream/monkey_patches/array.rb
+- lib/tensor_stream/monkey_patches/float.rb
 - lib/tensor_stream/monkey_patches/integer.rb
+- lib/tensor_stream/monkey_patches/patch.rb
 - lib/tensor_stream/nn/nn_ops.rb
 - lib/tensor_stream/operation.rb
 - lib/tensor_stream/ops.rb
@@ -293,6 +297,7 @@ files:
 - lib/tensor_stream/train/adagrad_optimizer.rb
 - lib/tensor_stream/train/adam_optimizer.rb
 - lib/tensor_stream/train/gradient_descent_optimizer.rb
+- lib/tensor_stream/train/learning_rate_decay.rb
 - lib/tensor_stream/train/momentum_optimizer.rb
 - lib/tensor_stream/train/optimizer.rb
 - lib/tensor_stream/train/rmsprop_optimizer.rb
@@ -315,6 +320,7 @@ files:
 - samples/multigpu.rb
 - samples/nearest_neighbor.rb
 - samples/raw_neural_net_sample.rb
+- samples/rnn.rb
 - tensor_stream.gemspec
 homepage: http://www.github.com/jedld/tensor_stream
 licenses: