tensor_stream 0.1.0

data/lib/tensor_stream/graph.rb

@@ -0,0 +1,98 @@
+module TensorStream
+  class Graph
+    attr_accessor :nodes, :collections, :eager_execution
+    
+    def initialize
+      @eager_execution = false
+      @nodes = {}
+      @collections = {
+        :"#{GraphKeys::GLOBAL_VARIABLES}" => []
+      }
+    end
+
+    def reset
+      @nodes = {}
+      @collections = {
+        :"#{GraphKeys::GLOBAL_VARIABLES}" => []
+      }
+    end
+
+    def self.get_default_graph
+      Thread.current[:tensor_stream_current_graph] || create_default
+    end
+
+    def self.create_default
+      Thread.current[:tensor_stream_current_graph] = TensorStream::Graph.new
+    end
+
+    def get_collection(name, options = {})
+      @collections[name.to_sym]
+    end
+
+    def add_to_collection(collection_name, val)
+      @collections[collection_name.to_sym] ||= []
+      @collections[collection_name.to_sym] << val
+    end
+
+    def add_node(node)
+      fail "Placeholder cannot be used when eager_execution is enabled" if @eager_execution && node.is_a?(Placeholder)
+      if @nodes[node.name]
+        node.name = uniqunify(node.name)
+      end
+
+      @nodes[node.name] = node
+      if @eager_execution
+        node.value = node.eval
+      end
+    end
+
+    def node_added?(name)
+      @nodes.key?(name)
+    end
+
+    def get_node(name)
+      @nodes[name]
+    end
+
+    def add_node!(name, node)
+      @nodes[name] = node
+      node
+    end
+
+    def add_variable(node, options = {})
+      fail "duplicate variable detected #{node.name} and reuse=false in current scope" if @nodes[node.name] && !options[:reuse]
+
+      add_to_collection(GraphKeys::GLOBAL_VARIABLES, node)
+
+      add_node(node)
+    end
+
+    def control_dependencies(dependencies = [], &block)
+      
+    end
+
+    def enable_eager_execution
+      @eager_execution = true
+    end
+
+    def disable_eager_execution
+      @eager_execution = false
+    end
+    
+    def executing_eagerly?
+      @eager_execution
+    end
+
+    protected
+
+    def uniqunify(name)
+      counter = 0
+      new_name = name
+      begin
+        counter +=1
+        new_name = "#{name}_#{counter}"
+      end while @nodes[new_name]
+      new_name
+    end
+  end
+end

data/lib/tensor_stream/graph_keys.rb

@@ -0,0 +1,5 @@
+module TensorStream
+  class GraphKeys
+    GLOBAL_VARIABLES = 'variables'
+  end
+end

data/lib/tensor_stream/helpers/op_helper.rb

@@ -0,0 +1,58 @@
+module TensorStream
+  module OpHelper
+    def op(code, a, b = nil, options = {})
+      Operation.new(code.to_sym, a, b, options)
+    end
+
+    # same as op but with a marker that it was internal generated
+    def i_op(code, a, b = nil, options = {})
+      Operation.new(code.to_sym, a, b, options.merge(internal: true))
+    end
+
+    def cons(value, options = {})
+      TensorStream.constant(value, options)
+    end
+
+    def i_cons(value, options = {})
+      TensorStream.constant(value, options.merge(internal: true))
+    end
+
+    def shape_eval(input)
+      return [] unless input.kind_of?(Array)
+      arr = []
+      arr_ptr = input
+
+      Kernel.loop do
+        arr << arr_ptr.size
+        arr_ptr = arr_ptr[0]
+
+        break unless arr_ptr.is_a?(Array)
+      end
+
+      arr
+    end
+
+  def dtype_eval(dtype, rank, value)
+    dtype = Tensor.detect_type(value[0])
+    rank+=1 if dtype == :array
+
+    [dtype, rank, value[0], value.size]
+  end
+
+  def val_to_dtype(value, rank = 0)
+    dtype = if value.is_a?(String)
+      :string
+    elsif value.is_a?(Float)
+      :float32
+    elsif value.is_a?(Integer)
+      :int32
+    elsif value.is_a?(Array)
+      rank += 1
+      :array
+    else
+      :float32
+    end
+    dtype
+  end
+  end
+end

data/lib/tensor_stream/math_gradients.rb

@@ -0,0 +1,161 @@
+module TensorStream
+  # Class that provides auto-differentiation
+  class MathGradients
+    extend TensorStream::OpHelper
+
+    def self.derivative(tensor, dx, options = {})
+      gradient_program_name = "_grad_#{tensor.name}_#{dx.name}"
+      return options[:graph].get_node(gradient_program_name) if options[:graph] && options[:graph].node_added?(gradient_program_name)
+
+      target_shape = options[:target_shape]
+      constant_options = { dtype: options[:dtype] }
+      constant_options_1 = { dtype: options[:dtype] || tensor.data_type, shape: target_shape }
+
+      return i_cons(1, constant_options_1) if tensor.equal?(dx)
+      return i_cons(0, constant_options) if options[:stop_gradients] && _include?(options[:stop_gradients], tensor)
+
+      if tensor.is_a?(Operation)
+        grad = derivative(tensor.items[0], dx, options) if tensor.items[0]
+        grad2 = derivative(tensor.items[1], dx, options) if tensor.items[1]
+
+        case tensor.operation
+        when :max
+          x_mask = i_op(:where, i_op(:ones_like, tensor.items[0]), i_op(:zeros_like, tensor.items[1]), pred: tensor.items[0] > tensor.items[1])
+          y_mask = i_op(:where, i_op(:zeros_like, tensor.items[0]), i_op(:ones_like, tensor.items[1]), pred: tensor.items[0] < tensor.items[1])
+          x_mask * grad + y_mask * grad2
+        when :where
+          x_mask = i_op(:where, i_op(:ones_like, tensor.items[0]), i_op(:zeros_like, tensor.items[1]), pred: tensor.options[:pred])
+          y_mask = i_op(:where, i_op(:zeros_like, tensor.items[0]), i_op(:ones_like, tensor.items[1]), pred: tensor.options[:pred])
+          x_mask * grad + y_mask * grad2
+        when :cond
+          i_op(:cond, grad, grad2, pred: tensor.options[:pred])
+        when :identity, :print, :pad
+          grad
+        when :negate
+          i_cons(-1, constant_options_1) * grad
+        when :abs
+          grad * i_op(:sign, _ds(tensor.items[0]))
+        when :square
+          i_cons(2, constant_options_1) * _ds(tensor.items[0]) * grad
+        when :exp
+          i_op(:exp, tensor.items[0]) * grad
+        when :log
+          (i_cons(1, constant_options_1) / _ds(tensor.items[0])) * grad
+        when :tanh
+          (i_cons(1, constant_options_1) - (i_op(:tanh, _ds(tensor.items[0]))**2)) * grad
+        when :tan
+          (i_cons(1, constant_options_1) / (i_op(:cos, _ds(tensor.items[0]))**2)) * grad
+        when :sin
+          i_op(:cos, tensor.items[0]) * grad
+        when :sqrt
+          i_cons(1, constant_options_1) / (i_cons(2, constant_options_1) * i_op(:sqrt, _ds(tensor.items[0]))) * grad
+        when :cos
+          -i_op(:sin, tensor.items[0]) * grad
+        when :add
+          grad_with_broadcast(tensor, dx, ->(a,b) { i_op(:add, a, b, name: 'grad_sum') } , options)
+        when :sub
+          grad_with_broadcast(tensor, dx, ->(a,b) { i_op(:sub, a, b, name: 'grad_sub') } , options)
+        when :pow
+          gx = _ds(tensor.items[1])*( _ds(tensor.items[0])**(_ds(tensor.items[1]) - 1)) * grad
+
+          log_x = i_op(:where, i_op(:log, tensor.items[0], nil, name: 'log_pow_grad'), i_op(:zeros_like, tensor.items[0]), pred: tensor.items[0] > 0)
+          gy = _ds(tensor.items[0])**_ds(tensor.items[1]) * log_x * grad2
+
+          gx + gy
+        when :div
+          # apply the quotient rule
+          gx = i_op(:div, grad, _ds(tensor.items[1]))
+          gy = grad2 * i_op(:div, i_op(:div, -_ds(tensor.items[0]), _ds(tensor.items[1])), _ds(tensor.items[1]))
+
+          gx + gy
+        when :mul
+          # apply the product rule
+          grad * _ds(tensor.items[1]) + _ds(tensor.items[0]) * grad2
+        when :reduce_mean
+          input_size = i_op(:reduce_prod, i_op(:shape, tensor.items[0]))
+          output_size = i_op(:reduce_prod, i_op(:shape, tensor))
+          factor = input_size / output_size
+ 
+          (grad / i_op(:cast, factor, data_type: grad.dtype))
+        when :reduce_sum
+          grad
+        when :stop_gradient
+          return i_cons(0, constant_options)
+        when :matmul
+          tensor_shape1 = tensor.items[1].shape ? tensor.items[1].shape.shape : nil
+          tensor_shape0 = tensor.items[0].shape ? tensor.items[0].shape.shape : nil
+
+          derivative_a = derivative(tensor.items[0], dx, target_shape: target_shape)
+          derivative_b = derivative(tensor.items[1], dx, target_shape: target_shape)
+
+          s0 =  i_op(:shape, tensor.items[0])
+          s1 =  i_op(:shape, tensor.items[1])
+
+          identity_0 = i_op(:ones, [s0[0], s1[1]], nil, data_type: tensor.items[0].data_type)
+          identity_1 = i_op(:ones, [s0[0], s1[1]], nil, data_type: tensor.items[1].data_type)
+
+          matmul_da = i_op(:matmul, identity_0, tensor.items[1], transpose_b: true,
+                                                     pad_zeros: true,
+                                                     name:        'matrix_dx')
+          matmul_db = i_op(:matmul, tensor.items[0], identity_1, transpose_a: true,
+                                                     pad_zeros: true,
+                                                     name:        'matrix_dy')       
+
+          zero_vect = i_op(:zeros, target_shape, nil, name: 'zero_vect')
+
+          # matmul_db = op(:transpose, matmul_db, nil).first
+
+          # begin_a = op(:zeros, op(:rank, matmul_db), nil, data_type: :int32, name: 'begin_a')
+          # matmul_b_shape = op(:shape, matmul_db)
+          # end_a = [matmul_b_shape[0], 1]
+
+          matmul_da = i_op(:cond, matmul_da[0], matmul_da, pred: op(:rank, derivative_a) > 0)
+
+          # matmul_da = op(:cond, matmul_da[0], matmul_da, pred: op(:rank, derivative_a) > 0)
+          norm_a = i_op(:mul, derivative_a, matmul_da, name: 'grad_a_norm_mul_da')
+          norm_b = i_op(:mul, derivative_b, matmul_db, name: 'grad_b_norm_mul_db')
+
+          # norm_a = i_op(:cond, norm_a[0], norm_a, pred: i_op(:rank, matmul_da) > i_op(:rank, derivative_a))
+          # norm_b = i_op(:cond, norm_b[0], norm_b, pred: i_op(:rank, matmul_db) > i_op(:rank, derivative_b))
+
+          i_op(:cond, norm_a, zero_vect, pred: i_op(:reduce_sum, norm_a) != 0) + i_op(:cond, norm_b, zero_vect, pred: i_op(:reduce_sum, norm_b) != 0)
+        else
+          raise "no derivative implementation found for op #{tensor.operation}"
+        end
+      elsif tensor.is_a?(TensorStream::Variable)
+        i_cons(0, constant_options)
+      elsif tensor.is_a?(TensorStream::Placeholder)
+        i_cons(0, constant_options)
+      else
+        i_cons(0, constant_options)
+      end.tap do |ops|
+        options[:graph].add_node!(gradient_program_name, ops) if options[:graph]
+      end
+    end
+
+    def self._ds(tensor)
+      return tensor unless tensor.is_a?(Operation)
+
+      case tensor.operation
+      when :reduce_sum
+        tensor.items[0]
+      else
+        tensor
+      end
+    end
+
+    def self.grad_with_broadcast(tensor, dx, func, options)
+      grad = derivative(tensor.items[0], dx, options)
+      grad2 = derivative(tensor.items[1], dx, options)
+      elements1 = i_op(:reduce_prod, i_op(:shape, tensor.items[0]), data_type: :float32)
+      elements2 = i_op(:reduce_prod, i_op(:shape, tensor.items[1]), data_type: :float32)
+      multiplier = elements1 / elements2
+      func.call(grad, grad2 * multiplier)
+    end
+
+    def self._include?(arr, obj)
+      arr.each { |a| return true if a.equal?(obj) }
+      false
+    end
+  end
+end

data/lib/tensor_stream/nn/nn_ops.rb

@@ -0,0 +1,17 @@
+module TensorStream
+  # High level machine learning functions
+  class NN
+    def self.softmax(logits, options = {})
+      TensorStream.exp(logits) / TensorStream.reduce_sum(TensorStream.exp(logits))
+    end
+
+    def self.relu(features, name: nil)
+      TensorStream.max(features, 0, name: "relu_#{name}")
+    end
+  end
+
+  # tensorflow compatibility
+  def self.nn
+    TensorStream::NN
+  end
+end

data/lib/tensor_stream/operation.rb

@@ -0,0 +1,195 @@
+module TensorStream
+  class Operation < Tensor
+    attr_accessor :name, :operation, :items, :rank, :options
+
+    def initialize(operation, a, b, options = {})
+      @operation = operation
+      @rank = options[:rank] || 0
+      @name = options[:name] || set_name
+      @internal = options[:internal]
+      @given_name = @name
+      @source = set_source(caller_locations)
+
+      @graph = options[:graph] || TensorStream.get_default_graph
+      @options = options
+
+
+      @items = [a, b].map { |i| options[:preserve_params_type] ? i : auto_wrap(i) }
+      @data_type = set_data_type(options[:data_type])
+
+      if options[:shape]
+        @shape = TensorShape.new(options[:shape], options[:shape].size || 0)
+      end
+      @graph.add_node(self)
+    end
+    def to_s
+      @name
+    end
+
+    def self.reset_counters
+      @@op_counter = 0
+    end
+
+    def to_h
+      {
+        op: operation,
+        name: name,
+        operands: hashify_tensor(items)
+      }
+    end
+
+    def self.empty_matrix?(m)
+      if m.kind_of?(Array)
+        m.each do |item|
+          if item.kind_of?(Array)
+            return false if !empty_matrix?(item)
+          else
+            return false if item!=0 || item!=0.0
+          end
+        end
+      end
+
+      return true
+    end
+
+    def set_data_type(passed_data_type)
+      case operation
+      when :greater, :less, :equal
+        :boolean
+      when :shape, :rank
+        :int32
+      else
+        passed_data_type || (@items[0] ? @items[0].data_type : :unknown)
+      end
+    end
+
+    def to_math(name_only = false, max_depth = 99)
+      return @name if max_depth == 0
+
+      sub_item = auto_math(items[0], name_only, max_depth - 1)
+
+      case operation
+      when :argmax
+        "argmax(#{auto_math(items[0])},#{options[:axis]})"
+      when :negate
+        "-#{sub_item}"
+      when :index
+        "#{sub_item}[#{auto_math(items[1], name_only, max_depth - 1)}]"
+      when :slice
+        "#{sub_item}[#{auto_math(items[1], name_only, max_depth - 1)}]"
+      when :assign_sub
+        "(#{items[0] ? items[0].name : "self"} -= #{auto_math(items[1], name_only)})"
+      when :assign_add
+        "(#{items[0] ? items[0].name : "self"} += #{auto_math(items[1], name_only)})"
+      when :assign
+        "(#{items[0] ? items[0].name : "self"} = #{auto_math(items[1], name_only)})"
+      when :sin, :cos, :tanh
+        "#{operation}(#{sub_item})"
+      when :add
+       "(#{sub_item} + #{auto_math(items[1], name_only, max_depth - 1)})"
+      when :sub
+        "(#{sub_item} - #{auto_math(items[1], name_only, max_depth - 1)})"
+      when :pow
+        "(#{sub_item}^#{auto_math(items[1], name_only, max_depth - 1)})"
+      when :div
+        "(#{sub_item} / #{auto_math(items[1], name_only, max_depth - 1)})"
+      when :mul
+        if auto_math(items[0]) == 1
+          auto_math(items[1], name_only, max_depth - 1)
+        elsif auto_math(items[1]) == 1
+          sub_item
+        else
+          "(#{sub_item} * #{auto_math(items[1], name_only, max_depth - 1)})"
+        end
+      when :reduce_sum
+        "reduce_sum(|#{sub_item}|)"
+      when :reduce_mean
+        "reduce_mean(|#{sub_item}|)"
+      when :reduce_prod
+        "reduce_prod(|#{sub_item}|)"
+      when :gradients
+        "gradient(#{sub_item})"
+      when :stop_gradient
+        sub_item
+      when :matmul
+        "#{sub_item}.matmul(#{auto_math(items[1], name_only, max_depth - 1)})"
+      when :eye
+        "eye(#{sub_item})"
+      when :transpose
+        "transpose(#{sub_item})"
+      when :shape
+        "#{sub_item}.shape"
+      when :exp
+        "e^#{sub_item})"
+      when :ones
+        "ones(#{sub_item})"
+      when :ones_like
+        "ones_like(#{sub_item})"
+      when :flow_group
+        "flow_group(#{items.collect { |i| auto_math(i)}.join(',')})"
+      when :zeros
+        "zeros(#{sub_item})"
+      when :reshape
+        "reshape(#{sub_item},#{auto_math(items[1], name_only, max_depth - 1)})"
+      when :rank
+        "#{sub_item}.rank"
+      when :cond
+        "(#{auto_math(options[:pred])} ? #{sub_item} : #{auto_math(items[1], name_only, max_depth - 1)})"
+      when :less
+        "#{sub_item} < #{auto_math(items[1], name_only, max_depth - 1)}"
+      when :greater
+        "#{sub_item} > #{auto_math(items[1], name_only, max_depth - 1)}"
+      when :square
+        "#{sub_item}\u00B2"
+      when :log
+        "log(#{sub_item})"
+      when :identity
+        "identity(#{sub_item})"
+      when :print
+        "print(#{sub_item})"
+      when :pad
+        "pad(#{sub_item},#{auto_math(options[:paddings])})"
+      when :equal
+        "#{sub_item} == #{auto_math(items[1], name_only, max_depth - 1)}"
+      when :not_equal
+        "#{sub_item} != #{auto_math(items[1], name_only, max_depth - 1)}"
+      when :sqrt
+        "sqrt(#{sub_item})"
+      when :zeros_like
+        "zeros_like(#{sub_item})"
+      when :where
+        "where(#{auto_math(options[:pred] , name_only, max_depth - 1)},#{auto_math(items[0])},#{auto_math(items[1])})"
+      when :max
+        "max(#{auto_math(sub_item)},#{auto_math(items[1])})"
+      when :cast
+        "cast(#{auto_math(sub_item)}, #{data_type})"
+      else
+        fail "math form for #{operation}"
+      end
+    end
+
+    def run
+      self.eval
+    end
+
+    private
+
+    def self.operation_counter
+      @@op_counter ||= 0
+
+      name = if @@op_counter == 0
+        ""
+      else
+        "_#{@@op_counter}"
+      end
+
+      @@op_counter += 1
+      
+      name
+    end
+
+    def set_name
+      "#{@operation}#{Operation.operation_counter}:#{@rank}"
+    end
+  end
+end