tensor_stream 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a25008802340137903308c5ec471a1ff4bedfc42
-  data.tar.gz: ba3549b627248f16d1afcea678d844473765d399
+  metadata.gz: f0d4f44991feb4af9b68cd9cd5ee6cd202bb2670
+  data.tar.gz: 64801dfe3a55d76d0d7535d24867f94fc4dd0861
 SHA512:
-  metadata.gz: 912fc4724db50b4b45a4cf7af11cd28a1df8dc9589a2347fb9c5374ada51cc34c11c179c70fc392ef0bd5c8c6dc41802d8ee4457df47cf7776ee78d92d1f0b74
-  data.tar.gz: 506e86fbd0877c10b51b6106529284c33c6158576e53f2fe720b85213f2ec1577156c614e85a587320e2d8a063dcefee51a59f6325cc1e24a27e8a2e99114ebf
+  metadata.gz: d68686258355308d9dbdd2fbd632a02fd98668765ab0ef3161494d28d32263aa6b88f4dbdc05da5b473c9f20cf8466bc64e93834442559cd2835a26ffe771c1c
+  data.tar.gz: 7517729b8aad2ffc2e557c3f55c09bf149fa84c4757b979842a3e56adb3e121331a73fc07667c5f8f639dd3c20b1c2f2f4b55311cf930f9d80cdffcea6a886e1

data/.circleci/config.yml

@@ -0,0 +1,57 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+       - image: circleci/ruby:2.4.1-node-browsers
+      
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile.lock" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
+            
+            bundle exec rspec -r rspec_junit_formatter --format progress \
+                            --format RspecJunitFormatter \
+                            --out /tmp/test-results/rspec.xml \
+                            --format progress \
+                            $TEST_FILES
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results

data/README.md

@@ -1,5 +1,7 @@
 [![Gem Version](https://badge.fury.io/rb/tensor_stream.svg)](https://badge.fury.io/rb/tensor_stream)
 
+[![CircleCI](https://circleci.com/gh/jedld/tensor_stream.svg?style=svg)](https://circleci.com/gh/jedld/tensor_stream)
+
 # TensorStream
 
 A reimplementation of TensorFlow for ruby. This is a ground up implementation with no dependency on TensorFlow. Effort has been made to make the programming style as near to TensorFlow as possible, comes with a pure ruby evaluator by default as well with support for an opencl evaluator.

data/lib/tensor_stream.rb

@@ -3,9 +3,11 @@ require 'deep_merge'
 require 'matrix'
 require 'concurrent'
 require 'tensor_stream/helpers/op_helper'
+require 'tensor_stream/initializer'
 require 'tensor_stream/graph_keys'
 require 'tensor_stream/types'
 require 'tensor_stream/graph'
+require 'tensor_stream/device'
 require 'tensor_stream/session'
 require 'tensor_stream/tensor_shape'
 require 'tensor_stream/tensor'
@@ -16,6 +18,8 @@ require 'tensor_stream/control_flow'
 require 'tensor_stream/trainer'
 require 'tensor_stream/nn/nn_ops'
 require 'tensor_stream/evaluator/evaluator'
+require 'tensor_stream/graph_serializers/pbtext'
+require 'tensor_stream/graph_serializers/graphml'
 # require 'tensor_stream/libraries/layers'
 require 'tensor_stream/monkey_patches/integer'
 require 'tensor_stream/ops'
@@ -29,6 +33,10 @@ module TensorStream
     Types.float32
   end
 
+  def self.graph
+    TensorStream::Graph.new
+  end
+
   def self.get_default_graph
     TensorStream::Graph.get_default_graph
   end
@@ -49,22 +57,60 @@ module TensorStream
     TensorStream::Graph.get_default_graph.executing_eagerly?
   end
 
-  def self.variable(value, options = {})
+  def self.variable(value, name: nil, initializer: nil, graph: nil, dtype: nil, trainable: true)
     common_options = {
-      initializer: Operation.new(:assign, nil, value),
-      name: options[:name]
+      initializer: initializer || Operation.new(:assign, nil, value),
+      name: name,
+      graph: graph,
+      dtype: dtype,
+      trainable: trainable
     }
     if value.is_a?(String)
-      TensorStream::Variable.new(options[:dtype] || :string, 0, [], common_options)
+      TensorStream::Variable.new(dtype || :string, 0, [], common_options)
     elsif value.is_a?(Integer)
-      TensorStream::Variable.new(options[:dtype] || :int32, 0, [], common_options)
+      TensorStream::Variable.new(dtype || :int32, 0, [], common_options)
     elsif value.is_a?(Float)
-      TensorStream::Variable.new(options[:dtype] || :float32, 0, [], common_options)
+      TensorStream::Variable.new(dtype || :float32, 0, [], common_options)
     else
-      TensorStream::Variable.new(options[:dtype] || :float32, 0, nil, common_options)
+      TensorStream::Variable.new(dtype || :float32, 0, nil, common_options)
+    end
+  end
+
+  def self.variable_scope(scope = nil, reuse: nil, initializer: nil)
+    Thread.current[:tensor_stream_variable_scope] ||= []
+    Thread.current[:tensor_stream_variable_scope] << OpenStruct.new(name: scope, reuse: reuse, initializer: initializer)
+    scope_name = __v_scope_name
+    begin
+      if block_given?
+        TensorStream.get_default_graph.name_scope(scope) do
+          yield(scope_name)
+        end
+      end
+    ensure
+      Thread.current[:tensor_stream_variable_scope].pop
     end
   end
 
+  def self.name_scope(name, default: nil, values: nil)
+    if values
+      graph_count = values.select { |v| v.is_a?(Tensor) }.map(&:graph).map(&:object_id).uniq.size
+      raise "values are not on the same graph" if graph_count > 1
+    end
+
+    get_default_graph.name_scope(name || default) do |scope|
+      yield scope if block_given?
+    end
+  end
+
+  def self.get_variable_scope
+    return nil unless Thread.current[:tensor_stream_variable_scope]
+    __v_scope_name
+  end
+
+  def self.__v_scope_name
+    Thread.current[:tensor_stream_variable_scope].map(&:name).compact.reject(&:empty?).join('/')
+  end
+
   def self.session(evaluator = :ruby_evaluator, thread_pool_class: Concurrent::ImmediateExecutor)
     session = TensorStream::Session.new(evaluator, thread_pool_class: thread_pool_class)
     yield session if block_given?
@@ -108,8 +154,8 @@ module TensorStream
     TensorStream::ControlFlow.new(:group, inputs)
   end
 
-  def self.get_variable(name, options = {})
-    TensorStream::Variable.new(options[:dtype] || :float32, nil, options[:shape], name: name, initializer: options[:initializer])
+  def self.get_variable(name, dtype: nil, shape: nil, initializer: nil, trainable: true, collections: nil)
+    TensorStream::Variable.new(dtype || :float32, nil, shape, collections: collections, name: name, initializer: initializer, trainable: trainable)
   end
 
   def self.get_collection(name, options = {})
@@ -128,10 +174,28 @@ module TensorStream
     TensorStream::Trainer
   end
 
+  def self.trainable_variables
+    TensorStream.get_default_graph.get_collection(TensorStream::GraphKeys::TRAINABLE_VARIABLES)
+  end
+
+  def self.set_random_seed(seed)
+    TensorStream.get_default_graph.random_seed = seed
+  end
+
+  def self.convert_to_tensor(value, dtype: nil, name: nil, preferred_dtype: nil)
+    return convert_to_tensor(value.call) if value.is_a?(Proc)
+
+    if !value.is_a?(Tensor)
+      i_cons(value, dtype: dtype || Tensor.detect_type(value), name: name)
+    else
+      value
+    end
+  end
+
   def self.check_allowed_types(input, types)
     return input unless input.is_a?(Tensor)
     return input if input.data_type.nil?
 
-    raise "Parameter data type #{input.data_type} passed not in #{types.join(',')}" unless types.map(&:to_sym).include?(input.data_type)
+    raise "#{input.source}: Parameter data type #{input.data_type} passed not in #{types.join(',')}" unless types.map(&:to_sym).include?(input.data_type)
   end
 end

data/lib/tensor_stream/control_flow.rb

@@ -8,10 +8,10 @@ module TensorStream
 
       @operation = :"flow_#{flow_type}"
       @items = items
-      @name = set_name
+      @name = [@graph.get_name_scope, options[:name] || set_name].compact.join('/')
       @ops = ops
       @source = format_source(caller_locations)
-
+      @shape = TensorShape.new([items.size])
       @graph.add_node(self)
     end
 

data/lib/tensor_stream/device.rb

@@ -0,0 +1,8 @@
+module TensorStream
+  class Device
+    attr_accessor :name
+    def initialize(name)
+      @name = name
+    end
+  end
+end

data/lib/tensor_stream/evaluator/ruby_evaluator.rb

@@ -1,6 +1,7 @@
 require 'tensor_stream/evaluator/operation_helpers/random_gaussian'
 require 'tensor_stream/evaluator/operation_helpers/array_ops_helper'
 require 'tensor_stream/math_gradients'
+require 'distribution'
 
 module TensorStream
   module Evaluator
@@ -28,19 +29,22 @@ module TensorStream
       include TensorStream::OpHelper
       include TensorStream::ArrayOpsHelper
 
-      def initialize(session, context, thread_pool: nil)
+      def initialize(session, context, thread_pool: nil, log_intermediates: false)
         @session = session
         @context = context
+        @log_intermediates = log_intermediates
         @retain = context[:retain] || []
         @thread_pool = thread_pool || Concurrent::ImmediateExecutor.new
+
+        @context[:compute_history] = [] if log_intermediates
       end
 
       def run(tensor, execution_context)
         return tensor.map { |t| run(t, execution_context) } if tensor.is_a?(Array)
 
         return tensor if retain.include?(tensor) # if var is in retain don't eval to value
-        
-        tensor = tensor.call() if tensor.is_a?(Proc)
+
+        tensor = tensor.call if tensor.is_a?(Proc)
 
         child_context = execution_context.dup
         res = if tensor.is_a?(Operation)
@@ -71,7 +75,9 @@ module TensorStream
       protected
 
       def eval_variable(tensor, child_context)
-        raise "variable #{tensor.name} not initalized" if tensor.value.nil?
+        if tensor.value.nil?
+          raise "variable #{tensor.name} not initalized"
+        end
         eval_tensor(tensor.value, child_context).tap do |val|
           child_context[:returns] ||= {}
           child_context[:returns][:vars] ||= []
@@ -86,6 +92,8 @@ module TensorStream
         b = resolve_placeholder(tensor.items[1], child_context) if tensor.items && tensor.items[1]
 
         case tensor.operation
+        when :const
+          complete_eval(a, child_context)
         when :argmax
           a = complete_eval(a, child_context)
           axis = tensor.options[:axis] || 0
@@ -150,6 +158,8 @@ module TensorStream
         when :concat
           values = complete_eval(a, child_context)
           concat_array(values, tensor.options[:axis])
+        when :round
+          call_op(:round, a, child_context, ->(t, _b) { t.round })
         when :abs
           call_op(:abs, a, child_context, ->(t, _b) { t.abs })
         when :tanh
@@ -162,27 +172,57 @@ module TensorStream
           call_op(:sin, a, child_context, ->(t, _b) { Math.sin(t) })
         when :cos
           call_op(:cos, a, child_context, ->(t, _b) { Math.cos(t) })
+        when :log1p
+          call_op(:log1p, a, child_context, ->(t, _b) { Distribution::MathExtension::Log.log1p(t) })
         when :log
           call_op(:log, a, child_context, ->(t, _b) { t < 0 ? Float::NAN : Math.log(t) })
         when :exp
           call_op(:exp, a, child_context, ->(t, _b) { Math.exp(t) })
+        when :sigmoid
+          call_op(:sigmoid, a, child_context, ->(t, _b) { 1 / (1 + Math.exp(-t)) })
         when :sqrt
           call_op(:exp, a, child_context, ->(t, _b) { Math.sqrt(t) })
         when :square
           call_op(:square, a, child_context, ->(t, _b) { t * t })
+        when :reciprocal
+          call_op(:square, a, child_context, ->(t, _b) { 1 / t })
         when :stop_gradient
           run(a, child_context)
         when :random_uniform
           maxval = tensor.options.fetch(:maxval, 1)
           minval = tensor.options.fetch(:minval, 0)
+          seed = tensor.options[:seed]
 
-          generator = -> { rand * (maxval - minval) + minval }
-          generate_vector(tensor.options[:shape], generator: generator)
+          random = _get_randomizer(tensor, seed)
+          generator = -> { random.rand * (maxval - minval) + minval }
+          shape = tensor.options[:shape] || tensor.shape.shape
+          generate_vector(shape, generator: generator)
         when :random_normal
-          r = RandomGaussian.new(tensor.options.fetch(:mean), tensor.options.fetch(:stddev))
+          random = _get_randomizer(tensor, seed)
+          r = RandomGaussian.new(tensor.options.fetch(:mean), tensor.options.fetch(:stddev), -> { random.rand })
+          random = _get_randomizer(tensor, seed)
           generator = -> { r.rand }
-
-          generate_vector(tensor.options[:shape], generator: generator)
+          shape = tensor.options[:shape] || tensor.shape.shape
+          generate_vector(shape, generator: generator)
+        when :glorot_uniform
+          random = _get_randomizer(tensor, seed)
+
+          shape = tensor.options[:shape] || tensor.shape.shape
+          fan_in, fan_out = if shape.size == 0
+                              [1, 1]
+                            elsif shape.size == 1
+                              [1, shape[0]]
+                            else
+                              [shape[0], shape.last]
+                            end
+
+          limit = Math.sqrt(6.0 / (fan_in + fan_out))
+
+          minval = -limit
+          maxval = limit
+
+          generator = -> { random.rand * (maxval - minval) + minval }
+          generate_vector(shape, generator: generator)
         when :flow_group
           tensor.items.collect { |item| run(item, child_context) }
         when :assign
@@ -314,8 +354,8 @@ module TensorStream
           rank_a = get_rank(matrix_a)
           rank_b = get_rank(matrix_b)
 
-          raise "#{a.name} rank must be greater than 1" if rank_a < 2
-          raise "#{b.name} rank must be greater than 1" if rank_b < 2
+          raise "#{tensor.items[0].name} rank must be greater than 1" if rank_a < 2
+          raise "#{tensor.items[1].name} rank must be greater than 1" if rank_b < 2
 
           matrix_a = matrix_a.transpose if tensor.options[:transpose_a]
           matrix_b = matrix_b.transpose if tensor.options[:transpose_b]
@@ -353,9 +393,9 @@ module TensorStream
           flat_arr = arr.flatten
           return flat_arr[0] if new_shape.size.zero? && flat_arr.size == 1
 
-          new_shape = fix_inferred_elements(new_shape, flat_arr.size)
+          new_shape = TensorShape.fix_inferred_elements(new_shape, flat_arr.size)
 
-          reshape(flat_arr, new_shape)
+          TensorShape.reshape(flat_arr, new_shape)
         when :pad
           a = complete_eval(a, child_context)
           p = complete_eval(tensor.options[:paddings], child_context)
@@ -380,19 +420,35 @@ module TensorStream
 
             tensor.breakpoint.call(tensor, a, b, complete_eval(result, child_context))
           end
+          if @log_intermediates
+            @context[:compute_history] << {
+              name: tensor.name,
+              type: tensor.data_type,
+              shape: shape_eval(result),
+              source: tensor.source,
+              description: tensor.to_math(true, 1),
+              value: result
+            }
+          end
           @context[tensor.name] = result
         end
       rescue EvaluatorExcecutionException => e
         raise e
       rescue StandardError => e
+        puts e.message
+        puts e.backtrace.join("\n")
+        shape_a = a.shape.shape if a
+        shape_b = b.shape.shape if b
+        dtype_a = a.data_type if a
+        dtype_b = b.data_type if b
         a = complete_eval(a, child_context)
         b = complete_eval(b, child_context)
         puts "name: #{tensor.given_name}"
         puts "op: #{tensor.to_math(true, 1)}"
-        puts "A: #{a}" if a
-        puts "B: #{b}" if b
+        puts "A #{shape_a} #{dtype_a}: #{a}" if a
+        puts "B #{shape_b} #{dtype_b}: #{b}" if b
+        dump_intermediates if @log_intermediates
 
-        puts e.backtrace.join("\n")
         raise EvaluatorExcecutionException.new(e, tensor), "error #{e.message} while evaluating #{tensor.name} : #{tensor.to_math(true,1)} defined at #{tensor.source}"
       end
 
@@ -515,30 +571,6 @@ module TensorStream
         end
       end
 
-      def fix_inferred_elements(shape, total_size)
-        return shape if shape.empty?
-
-        current_size = shape.inject(1) { |product, n| n > 0 ? product * n : product }
-        inferred_size = total_size / current_size
-        shape.map { |s| s == -1 ? inferred_size : s }
-      end
-
-      def reshape(arr, new_shape)
-        return arr if new_shape.empty?
-
-        s = new_shape.shift
-
-        if new_shape.size.zero?
-          raise "reshape dimen mismatch #{arr.size} != #{s}" if arr.size != s
-          return arr
-        end
-
-        dim = (arr.size / s)
-        arr.each_slice(dim).collect do |slice|
-          reshape(slice, new_shape.dup)
-        end
-      end
-
       def call_op(op, a, child_context, func)
         a = complete_eval(a, child_context)
         process_function_op(a, child_context, func)
@@ -723,6 +755,38 @@ module TensorStream
           generator.call
         end
       end
+
+      def _get_randomizer(tensor, seed)
+        if tensor.graph.random_seed && seed
+          Random.new(tensor.graph.random_seed ^ seed)
+        elsif tensor.graph.random_seed
+          @session.randomizer[tensor.graph.object_id] ||= Random.new(tensor.graph.random_seed)
+          @session.randomizer[tensor.graph.object_id]
+        elsif seed
+          @session.randomizer[tensor.operation] ||= Random.new(seed)
+          @session.randomizer[tensor.operation]
+        else
+          Random.new
+        end
+      end
+
+      def dump_intermediates
+        arr = []
+        arr << "============== start ==================="
+        @context[:compute_history].each_with_index do |history, index|
+          arr << "------------------------------------"
+          arr << history[:name]
+          arr << "#{history[:type]} #{history[:shape]}"
+          arr << history[:source]
+          arr << history[:description]
+          arr << ""
+          arr << history[:value].to_json
+          arr << "------------------------------------"
+        end
+        arr << "============== end ====================="
+        str = arr.join("\n")
+        File.write("/tmp/intermediates.txt", str)
+      end
     end
   end
 end