tensor_stream 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f0d4f44991feb4af9b68cd9cd5ee6cd202bb2670
-  data.tar.gz: 64801dfe3a55d76d0d7535d24867f94fc4dd0861
+SHA256:
+  metadata.gz: a5a6dce7a4317dee2e9fba536056d0d346ac9fd4a2763396a5735204f77b90c6
+  data.tar.gz: 4a6d2973badfa0f2ac20850f6885181c4f19b6fe00e416e9f8394f89ac267f77
 SHA512:
-  metadata.gz: d68686258355308d9dbdd2fbd632a02fd98668765ab0ef3161494d28d32263aa6b88f4dbdc05da5b473c9f20cf8466bc64e93834442559cd2835a26ffe771c1c
-  data.tar.gz: 7517729b8aad2ffc2e557c3f55c09bf149fa84c4757b979842a3e56adb3e121331a73fc07667c5f8f639dd3c20b1c2f2f4b55311cf930f9d80cdffcea6a886e1
+  metadata.gz: 285c9deb129680a9050a2afa9811de1d3cbcf2250cb78b4b6eab06f225bf8a5532d301a4cad7c0e9d39b7b16bcffb0527c7b6cadaad8b840a78cadc8fcf92c80
+  data.tar.gz: 2021fc72c95a8aad4e8b7f8a5b25fa7ce64e715ae08be9d3c325f058155f1a2ea9d58be38aae8f4849dabe067c75e140f174efe498503a86c7a73820dc367e5d

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.2.0] - 2018-05-27
+### Added
+- working logistic regression sample
+- reworked auto differentiation, fix a number of bugs related to auto differentiation, smaller derivative programs
+- alpha support for saving to pbtext format, added graphml generation
+- significant number of ops added
+- ops that support broadcasting now work better

data/README.md

@@ -102,6 +102,10 @@ You can take a look at spec/tensor_stream/operation_spec.rb for a list of suppor
 sliver of what TensorFlow can do, so feel free to file a PR to add requested
 ops and test cases.
 
+Other working samples can also be seen under tensor_stream/samples.
+
+Samples that are used for development and are still being made to work can be found under test_samples
+
 ## Python to Ruby guide
 
 Not all ops are available. Available ops are defined in lib/tensor_stream/ops.rb, corresponding gradients are found at lib/tensor_stream/math_gradients.
@@ -172,6 +176,36 @@ f = tf.matmul(a, b).breakpoint! { |tensor, a, b, result_value| binding.pry }
 tf.session.run(f)
 ```
 
+# Visualization
+
+tensorstream does not support tensorboard yet, but a graphml generator is included:
+
+```ruby
+tf = TensorStream
+a = tf.constant(1.0)
+b = tf.constant(2.0)
+result = a + b
+sess = tf.session
+sess.run(result)
+
+File.write('gradients.graphml', TensorStream::Graphml.new.get_string(result)) # dump graph only
+File.write('gradients.graphml', TensorStream::Graphml.new.get_string(result, sess)) # dump with values from session
+```
+
+the resulting graphml is designed to work with yED, after loading the graph change layout to "Flowchart" for best results
+
+## Exporting to TensorFlow
+
+Still in alpha but tensorstream supports TensorFlows as_graph_def serialization method:
+
+```ruby
+tf = TensorStream
+a = tf.constant(1.0)
+b = tf.constant(2.0)
+result = a + b
+File.write("model.pbtext", result.graph.as_graph_def)
+```
+
 ## Roadmap
 
 - Docs

data/lib/tensor_stream.rb

@@ -3,6 +3,7 @@ require 'deep_merge'
 require 'matrix'
 require 'concurrent'
 require 'tensor_stream/helpers/op_helper'
+require 'tensor_stream/helpers/string_helper'
 require 'tensor_stream/initializer'
 require 'tensor_stream/graph_keys'
 require 'tensor_stream/types'
@@ -18,8 +19,11 @@ require 'tensor_stream/control_flow'
 require 'tensor_stream/trainer'
 require 'tensor_stream/nn/nn_ops'
 require 'tensor_stream/evaluator/evaluator'
+require 'tensor_stream/graph_serializers/serializer'
 require 'tensor_stream/graph_serializers/pbtext'
 require 'tensor_stream/graph_serializers/graphml'
+require 'tensor_stream/math_gradients'
+
 # require 'tensor_stream/libraries/layers'
 require 'tensor_stream/monkey_patches/integer'
 require 'tensor_stream/ops'
@@ -150,8 +154,8 @@ module TensorStream
     end
   end
 
-  def self.group(inputs)
-    TensorStream::ControlFlow.new(:group, inputs)
+  def self.group(inputs, name: nil)
+    TensorStream::ControlFlow.new(:group, inputs, nil, name: name)
   end
 
   def self.get_variable(name, dtype: nil, shape: nil, initializer: nil, trainable: true, collections: nil)
@@ -196,6 +200,6 @@ module TensorStream
     return input unless input.is_a?(Tensor)
     return input if input.data_type.nil?
 
-    raise "#{input.source}: Parameter data type #{input.data_type} passed not in #{types.join(',')}" unless types.map(&:to_sym).include?(input.data_type)
+    raise "#{input.source}: Parameter data type #{input.data_type} passed not in #{types.join(',')}" unless types.include?(input.data_type.to_sym)
   end
 end

data/lib/tensor_stream/control_flow.rb

@@ -4,13 +4,12 @@ module TensorStream
     attr_accessor :ops
 
     def initialize(flow_type, items, ops = nil, options = {})
-      @graph = options[:graph] || TensorStream.get_default_graph
+      setup_initial_state(options)
 
       @operation = :"flow_#{flow_type}"
       @items = items
       @name = [@graph.get_name_scope, options[:name] || set_name].compact.join('/')
       @ops = ops
-      @source = format_source(caller_locations)
       @shape = TensorShape.new([items.size])
       @graph.add_node(self)
     end

data/lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb

@@ -1,6 +1,27 @@
 module TensorStream
   # varoius utility functions for array processing
   module ArrayOpsHelper
+    def slice_tensor(input, start, size)
+      start_index = start.shift
+      dimen_size = start_index + size.shift
+
+      input[start_index...dimen_size].collect do |item|
+        if item.is_a?(Array)
+          slice_tensor(item, start.dup, size.dup)
+        else
+          item
+        end
+      end
+    end
+
+    def truncate(input, target_shape)
+      rank = get_rank(input)
+      return input if rank.zero?
+
+      start = Array.new(rank) { 0 }
+      slice_tensor(input, start, target_shape)
+    end
+
     def broadcast(input_a, input_b)
       sa = shape_eval(input_a)
       sb = shape_eval(input_b)
@@ -24,7 +45,7 @@ module TensorStream
         input_b = broadcast_dimensions(input_b, target_shape)
       else
         target_shape = shape_diff(sb, sa)
-        raise "Incompatible shapes for op #{shape_eval(input_a)} vs #{shape_eval(input_a)}" if target_shape.nil?
+        raise "Incompatible shapes for op #{shape_eval(input_a)} vs #{shape_eval(input_b)}" if target_shape.nil?
 
         input_a = broadcast_dimensions(input_a, target_shape)
       end
@@ -52,7 +73,7 @@ module TensorStream
     end
 
     # handle 2 tensor math operations
-    def vector_op(vector, vector2, op = ->(a, b) { a + b }, switch = false)
+    def vector_op(vector, vector2, op = ->(a, b) { a + b }, switch = false, safe = true)
       if get_rank(vector) < get_rank(vector2) # upgrade rank of A
         duplicated = Array.new(vector2.size) do
           vector
@@ -65,6 +86,10 @@ module TensorStream
       vector.each_with_index.collect do |item, index|
         next vector_op(item, vector2, op, switch) if item.is_a?(Array) && get_rank(vector) > get_rank(vector2)
 
+        if safe && vector2.is_a?(Array)
+          next nil if vector2.size != 1 && index >= vector2.size
+        end
+
         z = if vector2.is_a?(Array)
               if index < vector2.size
                 vector2[index]
@@ -81,7 +106,7 @@ module TensorStream
         else
           switch ? op.call(z, item) : op.call(item, z)
         end
-      end
+      end.compact
     end
 
     def shape_diff(shape_a, shape_b)
@@ -96,5 +121,21 @@ module TensorStream
         s - reversed_b[index]
       end.reverse
     end
+
+    def tile_arr(input, dimen, multiples)
+      t = multiples[dimen]
+      if dimen == multiples.size - 1
+        return nil if t.zero?
+        input * t # ruby array dup
+      else
+        new_arr = input.collect do |sub|
+          tile_arr(sub, dimen + 1, multiples)
+        end.compact
+
+        return nil if new_arr.empty?
+
+        new_arr * t
+      end
+    end
   end
 end

data/lib/tensor_stream/evaluator/operation_helpers/math_helper.rb

@@ -0,0 +1,9 @@
+module TensorStream
+  # varoius utility functions for array processing
+  module MathHelper
+    # Calculates value of y = 1.0 / ( 1.0 + exp( -x ) )
+    def sigmoid(val)
+      1 / (1 + Math.exp(-val))
+    end
+  end
+end

data/lib/tensor_stream/evaluator/ruby_evaluator.rb

@@ -1,6 +1,6 @@
 require 'tensor_stream/evaluator/operation_helpers/random_gaussian'
 require 'tensor_stream/evaluator/operation_helpers/array_ops_helper'
-require 'tensor_stream/math_gradients'
+require 'tensor_stream/evaluator/operation_helpers/math_helper'
 require 'distribution'
 
 module TensorStream
@@ -28,6 +28,7 @@ module TensorStream
 
       include TensorStream::OpHelper
       include TensorStream::ArrayOpsHelper
+      include TensorStream::MathHelper
 
       def initialize(session, context, thread_pool: nil, log_intermediates: false)
         @session = session
@@ -179,7 +180,9 @@ module TensorStream
         when :exp
           call_op(:exp, a, child_context, ->(t, _b) { Math.exp(t) })
         when :sigmoid
-          call_op(:sigmoid, a, child_context, ->(t, _b) { 1 / (1 + Math.exp(-t)) })
+          call_op(:sigmoid, a, child_context, ->(t, _b) { sigmoid(t) })
+        when :sigmoid_grad
+          call_vector_op(:sigmoid_grad, a, b, child_context, ->(t, u) { u * sigmoid(t) * (1 - sigmoid(t)) })
         when :sqrt
           call_op(:exp, a, child_context, ->(t, _b) { Math.sqrt(t) })
         when :square
@@ -208,7 +211,7 @@ module TensorStream
           random = _get_randomizer(tensor, seed)
 
           shape = tensor.options[:shape] || tensor.shape.shape
-          fan_in, fan_out = if shape.size == 0
+          fan_in, fan_out = if shape.size.zero?
                               [1, 1]
                             elsif shape.size == 1
                               [1, shape[0]]
@@ -235,7 +238,7 @@ module TensorStream
         when :assign_sub
           tensor.items[0].value = process_vector_math_op(tensor.items[0], tensor.items[1], child_context, ->(t, u) { t - u })
           tensor.items[0].value
-        when :reduce_mean
+        when :mean
           c = fp_type?(tensor.data_type) ? 0.0 : 0
           func = lambda do |arr|
             return c if arr.nil?
@@ -249,7 +252,7 @@ module TensorStream
           end
 
           reduction(child_context, tensor, func)
-        when :reduce_sum
+        when :sum
           c = fp_type?(tensor.data_type) ? 0.0 : 0
           func = lambda do |arr|
             reduced_val = arr[0]
@@ -260,7 +263,10 @@ module TensorStream
           end
 
           reduction(child_context, tensor, func)
-        when :reduce_prod
+        when :tanh_grad
+          x = complete_eval(a, child_context)
+          call_op(:tanh_grad, x, child_context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
+        when :prod
           c = fp_type?(tensor.data_type) ? 1.0 : 1
           func = lambda do |arr|
             return c if arr.nil?
@@ -342,7 +348,15 @@ module TensorStream
             func.call
           else
             shape = [shape.to_i] unless shape.is_a?(Array)
-            generate_vector(shape, generator: func)
+
+            cache_key = "#{tensor.operation}_#{shape.to_s}"
+            if @context[:_cache].key?(cache_key)
+              return @context[:_cache][cache_key]
+            else
+              generate_vector(shape, generator: func).tap do |v|
+                @context[:_cache][cache_key] = v
+              end
+            end
           end
         when :shape
           input = complete_eval(a, child_context)
@@ -374,6 +388,10 @@ module TensorStream
           a = complete_eval(a, child_context)
           b = complete_eval(b, child_context)
           broadcast(a, b)
+        when :truncate
+          a = complete_eval(a, child_context)
+          b = complete_eval(b, child_context)
+          truncate(a, b)
         when :identity
           complete_eval(a, child_context)
         when :print
@@ -390,6 +408,8 @@ module TensorStream
           arr = complete_eval(a, child_context)
           new_shape = complete_eval(b, child_context)
 
+          arr = [arr] unless arr.is_a?(Array)
+
           flat_arr = arr.flatten
           return flat_arr[0] if new_shape.size.zero? && flat_arr.size == 1
 
@@ -411,6 +431,28 @@ module TensorStream
           b = complete_eval(b, child_context)
 
           get_broadcast_gradient_args(a, b)
+        when :reduced_shape
+          input_shape = complete_eval(a, child_context)
+          axes = complete_eval(b, child_context)
+
+          return [] if axes.nil? # reduce to scalar
+          axes = [ axes ] unless axes.is_a?(Array)
+          return input_shape if axes.empty?
+
+          axes.each do |dimen|
+            input_shape[dimen] = 1
+          end
+          input_shape
+        when :tile
+          input = complete_eval(a, child_context)
+          multiples = complete_eval(b, child_context)
+
+          rank = get_rank(input)
+          raise '1D or higher tensor required' if rank.zero?
+          raise "invalid multiple size passed #{rank} != #{multiples.size}" if rank != multiples.size
+
+          tile = tile_arr(input, 0, multiples)
+          tile.nil? ? [] : tile
         else
           raise "unknown op #{tensor.operation}"
         end.tap do |result|
@@ -437,18 +479,21 @@ module TensorStream
       rescue StandardError => e
         puts e.message
         puts e.backtrace.join("\n")
+
         shape_a = a.shape.shape if a
         shape_b = b.shape.shape if b
         dtype_a = a.data_type if a
         dtype_b = b.data_type if b
         a = complete_eval(a, child_context)
         b = complete_eval(b, child_context)
-        puts "name: #{tensor.given_name}"
-        puts "op: #{tensor.to_math(true, 1)}"
-        puts "A #{shape_a} #{dtype_a}: #{a}" if a
-        puts "B #{shape_b} #{dtype_b}: #{b}" if b
-        dump_intermediates if @log_intermediates
-
+        # puts "name: #{tensor.given_name}"
+        # # puts "op: #{tensor.to_math(true, 1)}"
+        # puts "A #{shape_a} #{dtype_a}: #{a}" if a
+        # puts "B #{shape_b} #{dtype_b}: #{b}" if b
+        # dump_intermediates if @log_intermediates
+        # File.write('/home/jedld/workspace/tensor_stream/samples/error.graphml', TensorStream::Graphml.new.get_string(tensor, @session))
+
+        # File.write('/Users/josephemmanueldayo/workspace/gradients.graphml', TensorStream::Graphml.new.get_string(tensor, @session))
         raise EvaluatorExcecutionException.new(e, tensor), "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true,1)} defined at #{tensor.source}"
       end
 
@@ -504,7 +549,7 @@ module TensorStream
 
       def reduction(child_context, tensor, func)
         val = complete_eval(tensor.items[0], child_context)
-        axis = complete_eval(tensor.options[:axis], child_context)
+        axis = complete_eval(tensor.items[1], child_context)
         keep_dims = complete_eval(tensor.options[:keepdims], child_context)
         rank = get_rank(val)
         return val if axis && axis.is_a?(Array) && axis.empty?
@@ -547,19 +592,6 @@ module TensorStream
         end
       end
 
-      def slice_tensor(input, start, size)
-        start_index = start.shift
-        dimen_size = start_index + size.shift
-
-        input[start_index...dimen_size].collect do |item|
-          if item.is_a?(Array)
-            slice_tensor(item, start.dup, size.dup)
-          else
-            item
-          end
-        end
-      end
-
       def matmul_const_transform(mat, mat_b, tensor)
         if !mat.is_a?(Array)
           compat_shape = shape_eval(mat_b).reverse
@@ -591,15 +623,17 @@ module TensorStream
         raise FullEvalNotPossible.new, "full eval not possible for #{a.name}" if eval_a.is_a?(Tensor) || eval_b.is_a?(Tensor)
 
         # ruby scalar
-        if get_rank(eval_a).zero?
-          if get_rank(eval_b).zero?
-            op.call(eval_a, eval_b)
-          else
-            vector_op(eval_b, eval_a, op, true)
-          end
-        elsif get_rank(eval_a) > 0
-          vector_op(eval_a, eval_b, op)
-        end
+        eval_a, eval_b = broadcast(eval_a, eval_b)
+        vector_op(eval_a, eval_b, op)
+        # if get_rank(eval_a).zero?
+        #   if get_rank(eval_b).zero?
+        #     op.call(eval_a, eval_b)
+        #   else
+        #     vector_op(eval_b, eval_a, op, true)
+        #   end
+        # else
+        #   vector_op(eval_a, eval_b, op)
+        # end
       end
 
       # determine possible reduction axis to be used

data/lib/tensor_stream/graph.rb

@@ -62,12 +62,14 @@ module TensorStream
       raise 'Placeholder cannot be used when eager_execution is enabled' if @eager_execution && node.is_a?(Placeholder)
 
       node.name = if @nodes[node.name]
-        uniqunify(node.name)
-      else
-        node.name
-      end
+                    uniqunify(node.name)
+                  else
+                    node.name
+                  end
 
       @nodes[node.name] = node
+
+      node.send(:propagate_outputs)
       node.send(:propagate_consumer, node)
       node.value = node.eval if @eager_execution
     end
@@ -89,14 +91,12 @@ module TensorStream
       scope = _variable_scope
 
       raise "duplicate variable detected #{node.name} and reuse=false in current scope" if @nodes[node.name] && !scope.reuse
-
       return @nodes[node.name] if @nodes[node.name]
-      
       raise "shape is not declared for #{node.name}" if node.shape.nil?
 
       if !options[:collections].nil? && !options[:collections].empty?
         options[:collections] = [options[:collections]] unless options[:collections].is_a?(Array)
-        options[:collections].each { |coll| add_to_collection(coll, node) } 
+        options[:collections].each { |coll| add_to_collection(coll, node) }
       end
 
       add_to_collection(GraphKeys::GLOBAL_VARIABLES, node)
@@ -162,6 +162,14 @@ module TensorStream
       graph_thread_storage[:current_scope].join('/')
     end
 
+    def as_graph_def
+      TensorStream::Pbtext.new.get_string(self)
+    end
+
+    def graph_def_versions
+      "producer: 26"
+    end
+
     protected
 
     def _variable_scope