tensor_stream 0.2.0 → 0.3.0

data/lib/tensor_stream/evaluator/opencl_template_helper.rb

@@ -0,0 +1,58 @@
+require 'erb'
+class OpenclTemplateHelper
+  def initialize(source)
+    @source = source
+  end
+
+  def generate
+    ERB.new(@source, nil, '%').result(binding)
+  end
+
+  def render(template, locals = {})
+    filename = File.join(File.dirname(__FILE__), 'kernels', "_#{template}")
+    source = File.read(filename)
+    current_scope = binding
+    locals.each do |k,v|
+      current_scope.local_variable_set(k.to_sym, v)
+    end
+    ERB.new(source, nil, '%').result(current_scope)
+  end
+
+  def dtype_to_c_type(dtype)
+    case(dtype)
+    when 'fp'
+      'float'
+    when 'int'
+      'int'
+    end
+  end
+
+  def operator_to_c(op)
+    case(op)
+    when 'less'
+      '<'
+    when 'less_equal'
+      '<='
+    when 'equal'
+      '=='
+    when 'greater'
+      '>'
+    when 'greater_equal'
+      '>='
+    when 'not_equal'
+      '!='
+    when 'logical_and'
+      '&&'
+    when 'div'
+      '/'
+    when 'add'
+      '+'
+    when 'sub'
+      '-'
+    when 'mul'
+      '*'
+    else
+      raise "unsupported op #{op}"
+    end
+  end
+end

data/lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb

@@ -22,6 +22,17 @@ module TensorStream
       slice_tensor(input, start, target_shape)
     end
 
+    def reduced_shape(input_shape, axes)
+      return [] if axes.nil? # reduce to scalar
+      axes = [ axes ] unless axes.is_a?(Array)
+      return input_shape if axes.empty?
+
+      axes.each do |dimen|
+        input_shape[dimen] = 1
+      end
+      input_shape
+    end
+
     def broadcast(input_a, input_b)
       sa = shape_eval(input_a)
       sb = shape_eval(input_b)
@@ -137,5 +148,21 @@ module TensorStream
         new_arr * t
       end
     end
+
+    def process_function_op(a, op)
+      # ruby scalar
+      if (a.is_a?(Tensor) && a.shape.rank > 0) || a.is_a?(Array)
+        vector_op(a, 0, op)
+      else
+        op.call(a, 0)
+      end
+    end
+
+    def get_rank(value, rank = 0)
+      return rank unless value.is_a?(Array)
+      return rank + 1 if value.empty?
+
+      get_rank(value[0], rank + 1)
+    end
   end
 end

data/lib/tensor_stream/evaluator/ruby_evaluator.rb

@@ -76,10 +76,11 @@ module TensorStream
       protected
 
       def eval_variable(tensor, child_context)
-        if tensor.value.nil?
+        value = tensor.read_value
+        if value.nil?
           raise "variable #{tensor.name} not initalized"
         end
-        eval_tensor(tensor.value, child_context).tap do |val|
+        eval_tensor(value, child_context).tap do |val|
           child_context[:returns] ||= {}
           child_context[:returns][:vars] ||= []
           child_context[:returns][:vars] << { name: tensor.name, val: val }
@@ -99,7 +100,12 @@ module TensorStream
           a = complete_eval(a, child_context)
           axis = tensor.options[:axis] || 0
 
-          get_max_with_axis(a, axis, 0, tensor.data_type)
+          get_op_with_axis(a, axis, 0, tensor.data_type)
+        when :argmin
+          a = complete_eval(a, child_context)
+          axis = tensor.options[:axis] || 0
+
+          get_op_with_axis(a, axis, 0, tensor.data_type, ->(a, b) { a < b })
         when :cast
           a = complete_eval(a, child_context)
 
@@ -499,7 +505,10 @@ module TensorStream
 
       def eval_tensor(tensor, child_context)
         return tensor unless tensor.is_a?(Tensor)
-        return @context[tensor.name] if @context.key?(tensor.name)
+
+        cache_key = "#{tensor.graph.object_id}_ruby_#{tensor.name}"
+        return @context[cache_key] if @context.key?(cache_key)
+        return @context[:_cache][cache_key] if @context[:_cache] && @context[:_cache].key?(tensor.name)
 
         if tensor.value.is_a?(Array)
           tensor.value.collect do |item|
@@ -508,20 +517,21 @@ module TensorStream
         else
           tensor.value.is_a?(Tensor) ? run(tensor.value, child_context) : tensor.value
         end.tap do |result|
-          @context[tensor.name] = result
+          @context[cache_key] = result
+          @context[:_cache][cache_key] = result if @context[:_cache] && tensor.is_const
         end
       end
 
       private
 
-      def get_max_with_axis(a, target_axis, current_axis, output_type)
+      def get_op_with_axis(a, target_axis, current_axis, output_type, op = ->(t, u) { t > u })
         if target_axis == current_axis
           if a[0].is_a?(Array)
             (0...a[0].size).each.collect do |column_index|
               max = nil
               max_index = 0
               a.each_with_index do |row, row_index|
-                if max.nil? || row[column_index] > max
+                if max.nil? || op.call(row[column_index], max)
                   max = row[column_index]
                   max_index = row_index
                 end
@@ -533,7 +543,7 @@ module TensorStream
             max = nil
             max_index = 0
             a.each_with_index do |x, index|
-              if max.nil? || x > max
+              if max.nil? || op.call(x, max)
                 max = x
                 max_index = index
               end
@@ -542,7 +552,7 @@ module TensorStream
           end
         else
           a.collect do |row|
-            get_max_with_axis(row, target_axis, current_axis + 1, output_type)
+            get_op_with_axis(row, target_axis, current_axis + 1, output_type, op)
           end
         end
       end
@@ -605,7 +615,7 @@ module TensorStream
 
       def call_op(op, a, child_context, func)
         a = complete_eval(a, child_context)
-        process_function_op(a, child_context, func)
+        process_function_op(a, func)
       rescue FullEvalNotPossible
         TensorStream.send(op.to_sym, a)
       end
@@ -667,13 +677,6 @@ module TensorStream
         end
       end
 
-      def get_rank(value, rank = 0)
-        return rank unless value.is_a?(Array)
-        return rank + 1 if value.empty?
-
-        get_rank(value[0], rank + 1)
-      end
-
       def concat_array(values, axis)
         combined_array = values.shift
         axis = get_rank(combined_array) - 1 if axis == -1
@@ -694,20 +697,6 @@ module TensorStream
         end
       end
 
-      def process_function_op(a, child_context, op)
-        # ruby scalar
-        if (a.is_a?(Tensor) && a.shape.rank > 0) || a.is_a?(Array)
-          vector_op(a, 0, op)
-        elsif !a.is_a?(Tensor) || a.shape.rank.zero?
-          v = run(a, child_context)
-          raise FullEvalNotPossible.new, "full eval not possible for #{v.name}" if v.is_a?(Tensor) && !v.is_const
-
-          op.call(v, 0)
-        else
-          raise 'cannot be here'
-        end
-      end
-
       def resolve_placeholder(placeholder, _execution_context = {})
         return nil if placeholder.nil?
         return placeholder if retain.include?(placeholder)

data/lib/tensor_stream/graph.rb

@@ -1,7 +1,7 @@
 module TensorStream
   # A class that defines a TensorStream graph
   class Graph
-    attr_accessor :nodes, :collections, :eager_execution, :random_seed
+    attr_accessor :nodes, :collections, :eager_execution, :random_seed, :constants
 
     def initialize
       @eager_execution = false
@@ -9,6 +9,7 @@ module TensorStream
       @collections = {
         :"#{GraphKeys::GLOBAL_VARIABLES}" => []
       }
+      @constants = {}
     end
 
     def reset
@@ -21,6 +22,7 @@ module TensorStream
       @collections = {
         :"#{GraphKeys::GLOBAL_VARIABLES}" => []
       }
+      @constants = {}
     end
 
     def as_default
@@ -68,7 +70,7 @@ module TensorStream
                   end
 
       @nodes[node.name] = node
-
+      @constants[node.name] = node if node.is_const
       node.send(:propagate_outputs)
       node.send(:propagate_consumer, node)
       node.value = node.eval if @eager_execution

data/lib/tensor_stream/math_gradients.rb

@@ -188,6 +188,9 @@ module TensorStream
         when :zeros_like
           # non differentiable
           nil
+        when :argmin, :argmax
+          # non differentiable
+          [nil, nil]
         else
           raise "no derivative op for #{node.operation}"
         end

data/lib/tensor_stream/operation.rb

@@ -17,7 +17,7 @@ module TensorStream
 
       @items = [input_a, input_b].map { |i| options[:preserve_params_type] ? i : TensorStream.convert_to_tensor(i) }
       @data_type = set_data_type(options[:data_type])
-
+      @is_const = infer_const
       @shape = TensorShape.new(infer_shape)
       @graph.add_node(self)
     end
@@ -48,14 +48,39 @@ module TensorStream
       true
     end
 
+    def infer_const
+      return false if breakpoint
+      case operation
+      when :random_normal, :random_uniform, :glorot_uniform, :print
+        false
+      else
+        non_const = @items.compact.find { |item| !item.is_const }
+        non_const ? false : true
+      end
+    end
+
     def set_data_type(passed_data_type)
       case operation
-      when :greater, :less, :equal, :not_equal, :greater_equal, :less_equal
+      when :greater, :less, :equal, :not_equal, :greater_equal, :less_equal, :logical_and
         :boolean
       when :shape, :rank
         :int32
+      when :random_normal, :random_uniform, :glorot_uniform
+        passed_data_type || :float32
+      when :index
+        if @items[0].is_a?(ControlFlow)
+
+          if @items[1].is_const
+            @items[0].items[@items[1].value].data_type
+          else
+            :unknown
+          end
+        else
+          @items[0].data_type
+        end
       else
         return passed_data_type if passed_data_type
+
         if @items[0]
           @items[0].data_type
         elsif @items[1]
@@ -228,6 +253,8 @@ module TensorStream
         return []
       when :zeros, :ones
         return items[0] ? items[0].value : options[:shape]
+      when :zeros_like, :ones_like
+        items[0].shape.shape
       when :shape
         return items[0].shape.shape ? [items[0].shape.shape.size] : nil
       when :matmul

data/lib/tensor_stream/ops.rb

@@ -9,6 +9,10 @@ module TensorStream
       _op(:argmax, input, nil, axis: axis, name: name, dimension: dimension, data_type: output_type)
     end
 
+    def argmin(input, axis = nil, name: nil, dimension: nil, output_type: :int32)
+      _op(:argmin, input, nil, axis: axis, name: name, dimension: dimension, data_type: output_type)
+    end
+
     def gradients(input, wrt_xs, grad_ys: nil,
       name: 'gradients',
       colocate_gradients_with_ops: false,
@@ -157,6 +161,10 @@ module TensorStream
       _op(:sub, input_a, input_b, name: name)
     end
 
+    def subtract(input_a, input_b, name: nil)
+      sub(input_a, input_b, name: name)
+    end
+
     def max(input_a, input_b, name: nil)
       check_allowed_types(input_a, NUMERIC_TYPES)
       check_allowed_types(input_b, NUMERIC_TYPES)
@@ -176,8 +184,12 @@ module TensorStream
       _op(:print, input, data, message: message, name: name)
     end
 
-    def negate(input, options = {})
-      _op(:negate, input, nil, options)
+    def negate(input, name: nil)
+      _op(:negate, input, nil, name: name)
+    end
+
+    def negative(input, name: nil)
+      negate(input, name: name)
     end
 
     def equal(input_a, input_b, name: nil)

data/lib/tensor_stream/placeholder.rb

@@ -4,7 +4,7 @@ module TensorStream
     def initialize(data_type, rank, shape, options = {})
       setup_initial_state(options)
 
-      @data_type = data_type
+      @data_type = data_type.to_sym
       @rank = rank
       @shape = TensorShape.new(shape, rank)
       @value = nil

data/lib/tensor_stream/session.rb

@@ -3,15 +3,20 @@ module TensorStream
   class Session
     include StringHelper
 
-    attr_reader :last_session_context, :closed, :target
+    attr_reader :last_session_context, :closed, :target, :session_cache
     attr_accessor :randomizer
 
-    def initialize(evaluator = :ruby_evaluator, thread_pool_class: Concurrent::ImmediateExecutor)
+    def initialize(evaluator = :ruby_evaluator, thread_pool_class: Concurrent::ImmediateExecutor, evaluator_options: {})
       @evaluator_class = Object.const_get("TensorStream::Evaluator::#{camelize(evaluator.to_s)}")
       @thread_pool = thread_pool_class.new
       @closed = false
       @session_cache = {}
       @randomizer = {}
+      @evaluator_options = evaluator_options
+    end
+
+    def clear_session_cache
+      @session_cache = {}
     end
 
     def self.default_session
@@ -37,7 +42,9 @@ module TensorStream
         end
       end
 
-      evaluator = @evaluator_class.new(self, context.merge!(retain: options[:retain]), thread_pool: @thread_pool, log_intermediates: options[:log_intermediates])
+      @evaluator_options[:thread_pool] = @thread_pool
+      @evaluator_options[:log_intermediates] = options[:log_intermediates]
+      evaluator = @evaluator_class.new(self, context.merge!(retain: options[:retain]), @evaluator_options)
 
       execution_context = {}
       @last_session_context = context

data/lib/tensor_stream/tensor_shape.rb

@@ -51,7 +51,7 @@ module TensorStream
 
     def self.reshape(arr, new_shape)
       return arr if new_shape.empty?
-
+      new_shape = new_shape.dup
       s = new_shape.shift
 
       if new_shape.size.zero?

data/lib/tensor_stream/train/saver.rb

@@ -22,7 +22,7 @@ module TensorStream
         }
 
         vars.each do |variable|
-          variables[variable.name] = variable.value
+          variables[variable.name] = variable.read_value
         end
 
         basename = File.basename(outputfile)

data/lib/tensor_stream/variable.rb

@@ -1,7 +1,7 @@
 module TensorStream
   # Class that defines a TensorStream variable
   class Variable < Tensor
-    attr_accessor :trainable, :options
+    attr_accessor :trainable, :options, :buffer
     def initialize(data_type, rank, shape, options = {})
       setup_initial_state(options)
 
@@ -10,6 +10,7 @@ module TensorStream
       @data_type = data_type
       @rank = rank
       @value = nil
+      @is_const = false
       @name = [TensorStream.get_variable_scope, options[:name] || build_name].compact.reject(&:empty?).join('/')
       @initalizer_tensor = options[:initializer] ? options[:initializer] : _variable_scope.initializer || TensorStream.glorot_uniform_initializer
       if shape.nil? && @initalizer_tensor && @initalizer_tensor.shape
@@ -36,6 +37,11 @@ module TensorStream
     end
 
     def read_value
+      if buffer && buffer.dirty
+        @value = buffer.to_ruby
+        buffer.dirty = false
+      end
+
       @value
     end