tensor_stream 0.1.4 → 0.1.5

data/lib/tensor_stream/graph.rb

@@ -1,7 +1,7 @@
 module TensorStream
   # A class that defines a TensorStream graph
   class Graph
-    attr_accessor :nodes, :collections, :eager_execution
+    attr_accessor :nodes, :collections, :eager_execution, :random_seed
 
     def initialize
       @eager_execution = false
@@ -16,13 +16,31 @@ module TensorStream
       @const_counter = 0
       @var_counter = 0
       @op_counter = 0
-      
+      @random_seed = nil
       @nodes = {}
       @collections = {
         :"#{GraphKeys::GLOBAL_VARIABLES}" => []
       }
     end
 
+    def as_default
+      Thread.current[:tensor_stream_current_graph] = self
+      yield(self) if block_given?
+      self
+    end
+
+    def name_scope(name = nil)
+      Thread.current["ts_graph_#{object_id}"] ||= {}
+      Thread.current["ts_graph_#{object_id}"][:current_scope] ||= []
+      Thread.current["ts_graph_#{object_id}"][:current_scope] << name
+
+      begin
+        yield get_name_scope if block_given?
+      ensure
+        Thread.current["ts_graph_#{object_id}"][:current_scope].pop
+      end
+    end
+
     def self.get_default_graph
       Thread.current[:tensor_stream_current_graph] || create_default
     end
@@ -42,7 +60,13 @@ module TensorStream
 
     def add_node(node)
       raise 'Placeholder cannot be used when eager_execution is enabled' if @eager_execution && node.is_a?(Placeholder)
-      node.name = uniqunify(node.name) if @nodes[node.name]
+
+      node.name = if @nodes[node.name]
+        uniqunify(node.name)
+      else
+        node.name
+      end
+
       @nodes[node.name] = node
       node.send(:propagate_consumer, node)
       node.value = node.eval if @eager_execution
@@ -62,10 +86,21 @@ module TensorStream
     end
 
     def add_variable(node, options = {})
-      raise "duplicate variable detected #{node.name} and reuse=false in current scope" if @nodes[node.name] && !options[:reuse]
+      scope = _variable_scope
 
-      add_to_collection(GraphKeys::GLOBAL_VARIABLES, node)
+      raise "duplicate variable detected #{node.name} and reuse=false in current scope" if @nodes[node.name] && !scope.reuse
 
+      return @nodes[node.name] if @nodes[node.name]
+      
+      raise "shape is not declared for #{node.name}" if node.shape.nil?
+
+      if !options[:collections].nil? && !options[:collections].empty?
+        options[:collections] = [options[:collections]] unless options[:collections].is_a?(Array)
+        options[:collections].each { |coll| add_to_collection(coll, node) } 
+      end
+
+      add_to_collection(GraphKeys::GLOBAL_VARIABLES, node)
+      add_to_collection(GraphKeys::TRAINABLE_VARIABLES, node) if node.trainable?
       add_node(node)
     end
 
@@ -120,8 +155,21 @@ module TensorStream
       name
     end
 
+    def get_name_scope
+      graph_thread_storage = Thread.current["ts_graph_#{object_id}"]
+      return nil if graph_thread_storage.nil?
+
+      graph_thread_storage[:current_scope].join('/')
+    end
+
     protected
 
+    def _variable_scope
+      return OpenStruct.new(name: '', reuse: false, initializer: nil) if Thread.current[:tensor_stream_variable_scope].nil? || Thread.current[:tensor_stream_variable_scope].empty?
+      scope = Thread.current[:tensor_stream_variable_scope].last
+      scope
+    end
+
     def uniqunify(name)
       counter = 0
       new_name = name

data/lib/tensor_stream/graph_keys.rb

@@ -1,5 +1,6 @@
 module TensorStream
   class GraphKeys
     GLOBAL_VARIABLES = 'variables'.freeze
+    TRAINABLE_VARIABLES = 'trainable_variables'.freeze
   end
 end

data/lib/tensor_stream/graph_serializers/graphml.rb

@@ -0,0 +1,91 @@
+module TensorStream
+  class Graphml
+    def initialize
+    end
+
+    def serialize(session, tensor, filename)
+      @session = session
+      @last_session_context = session.last_session_context
+
+      arr_buf = []
+      arr_buf << '<?xml version="1.0" encoding="UTF-8"?>'
+      arr_buf << '<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+      xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">'
+      arr_buf << '<key id="d0" for="node" attr.name="label" attr.type="string"/>'
+      arr_buf << '<key id="d1" for="node" attr.name="formula" attr.type="string"/>'
+      arr_buf << '<key id="d2" for="node" attr.name="color" attr.type="string"/>'
+      arr_buf << '<key id="d3" for="node" attr.name="value" attr.type="string"/>'
+      arr_buf << "<graph id=\"g_#{_gml_string(tensor.name)}\" edgedefault=\"directed\">"
+      arr_buf << "<node id=\"out\">"
+      arr_buf << "<data key=\"d0\">out</data>"
+      arr_buf << "<data key=\"d2\">red</data>"
+      arr_buf << "</node>"
+      to_graph_ml(tensor, arr_buf)
+      arr_buf << "<edge source=\"#{_gml_string(tensor.name)}\" target=\"out\"/>"
+      arr_buf << "</graph>"
+      arr_buf << "</graphml>"
+      File.write(filename, arr_buf.join("\n"))
+    end
+
+    private
+
+    def _val(tensor)
+      JSON.pretty_generate(@last_session_context[tensor.name])
+    end
+
+    def to_graph_ml(tensor, arr_buf = [], added = {}, _id = 0)
+      puts tensor.name
+      added[tensor.name] = true
+      arr_buf << "<node id=\"#{_gml_string(tensor.name)}\">"
+      arr_buf << "<data key=\"d0\">#{tensor.operation}</data>"
+      arr_buf << "<data key=\"d1\">#{tensor.to_math(true, 1)}</data>"
+      arr_buf << "<data key=\"d2\">blue</data>"
+      if @last_session_context[tensor.name]
+        arr_buf << "<data key=\"d3\">#{_val(tensor)}</data>"
+      end
+      arr_buf << "</node>"
+
+      tensor.items.each do |item|
+        next unless item
+        next if _added[item.name]
+
+        next to_graph_ml(item, arr_buf, added) if item.is_a?(Operation)
+        added[item.name] = true
+        if item.is_a?(Variable)
+          arr_buf << "<node id=\"#{_gml_string(item.name)}\">"
+          arr_buf << "<data key=\"d0\">#{item.name}</data>"
+          arr_buf << "<data key=\"d2\">green</data>"
+          if @last_session_context[item.name]
+            arr_buf << "<data key=\"d3\">#{_val(tensor)}</data>"
+          end
+          arr_buf << "</node>"
+        elsif item.is_a?(Placeholder)
+          arr_buf << "<node id=\"#{_gml_string(item.name)}\">"
+          arr_buf << "<data key=\"d0\">#{item.name}</data>"
+          arr_buf << "<data key=\"d2\">yellow</data>"
+          if @last_session_context[item.name]
+            arr_buf << "<data key=\"d3\">#{_val(tensor)}</data>"
+          end
+          arr_buf << "</node>"
+        else
+          arr_buf << "<node id=\"#{_gml_string(item.name)}\">"
+          arr_buf << "<data key=\"d0\">#{item.name}</data>"
+          arr_buf << "<data key=\"d2\">black</data>"
+          if @last_session_context[item.name]
+            arr_buf << "<data key=\"d3\">#{_val(tensor)}</data>"
+          end
+          arr_buf << "</node>"
+        end
+      end
+
+      tensor.items.each do |item|
+        next unless item
+        arr_buf << "<edge source=\"#{_gml_string(item.name)}\" target=\"#{_gml_string(tensor.name)}\"/>"
+      end
+    end
+
+    def _gml_string(str)
+      str.gsub('/','-')
+    end
+  end
+end

data/lib/tensor_stream/graph_serializers/pbtext.rb

@@ -0,0 +1,71 @@
+module TensorStream
+  class Pbtext
+    def initialize
+    end
+
+    def serialize(session, filename, tensor)
+    end
+
+    def get_string(graph)
+      @lines = []
+      graph.nodes.each do |k, node|
+        @lines << "node {"
+        @lines << "  name: #{node.name.to_json}"
+        if node.is_a?(TensorStream::Operation)
+          @lines << "  op: #{node.operation.to_json}"
+          node.items.each do |input|
+            next unless input
+            @lines << "  input: #{input.name.to_json}"
+          end
+          # type
+          pb_attr('T', sym_to_protobuf_type(node.data_type))
+        elsif node.is_a?(TensorStream::Tensor) && node.is_const
+          @lines << "  op: \"Const\""
+          # type
+          pb_attr('T', sym_to_protobuf_type(node.data_type))
+          pb_attr('value', tensor_value(node))
+        end
+        @lines << "}"
+      end
+      @lines.join("\n")
+    end
+
+    private
+
+    def tensor_value(tensor)
+      arr = []
+      arr << "tensor {"
+      arr << "  dtype: #{sym_to_protobuf_type(tensor.data_type)}"
+      arr << "  float_val: #{tensor.value}"
+      arr << "}"
+      arr
+    end
+
+    def sym_to_protobuf_type(type)
+      case type
+      when :int32
+        "DT_INT32"
+      when :float, :float32
+        "DT_FLOAT"
+      else
+        "DT_UNKNOWN"
+      end
+    end
+
+    def pb_attr(key, value)
+      @lines << "  attr {"
+      @lines << "    key: \"#{key}\""
+      @lines << "    value {"
+      if value.is_a?(Array)
+        value.each do |v|
+          @lines << "      #{v}"
+        end
+      else
+        @lines << "      #{value}"
+      end
+      @lines << "    }"
+      @lines << "  }"
+    end
+  end
+
+end

data/lib/tensor_stream/helpers/op_helper.rb

@@ -1,7 +1,7 @@
 module TensorStream
   # module that contains helper functions useful for ops
   module OpHelper
-    def op(code, t_a, t_b = nil, options = {})
+    def _op(code, t_a, t_b = nil, options = {})
       Operation.new(code.to_sym, t_a, t_b, options)
     end
 
@@ -58,5 +58,11 @@ module TensorStream
     def fp_type?(type)
       TensorStream::Ops::FLOATING_POINT_TYPES.include?(type)
     end
+
+    def format_source(trace)
+      grad_source = trace.select { |c| c.to_s.include?(File.join('lib', 'tensor_stream', 'math_gradients')) }.first
+      source = trace.reject { |c| c.to_s.include?(File.join('lib', 'tensor_stream')) }.first
+      [grad_source, source].compact.join("\n")
+    end
   end
 end

data/lib/tensor_stream/initializer.rb

@@ -0,0 +1,16 @@
+module TensorStream
+  class Initializer
+    attr_writer :op
+    def initialize(op)
+      @op = op
+    end
+
+    def op
+      @op.call
+    end
+
+    def shape
+      nil
+    end
+  end
+end

data/lib/tensor_stream/math_gradients.rb

@@ -5,6 +5,7 @@ module TensorStream
 
     def self.derivative(tensor, wrt_dx, options = {})
       gradient_program_name = "_grad_#{tensor.name}_#{wrt_dx.name}"
+      
       return options[:graph].get_node(gradient_program_name) if options[:graph] && options[:graph].node_added?(gradient_program_name)
 
       constant_options = { dtype: options[:dtype] }
@@ -18,6 +19,10 @@ module TensorStream
         grad2 = derivative(tensor.items[1], wrt_dx, options) if tensor.items[1]
 
         case tensor.operation
+        when :zeros_like
+          i_cons(0, constant_options)
+        when :log1p
+          grad * _op(:reciprocal, i_cons(1, constant_options_1) + tensor.items[0])
         when :max
           x_mask = i_op(:where, i_op(:ones_like, tensor.items[0]), i_op(:zeros_like, tensor.items[1]), pred: tensor.items[0] > tensor.items[1])
           y_mask = i_op(:where, i_op(:zeros_like, tensor.items[0]), i_op(:ones_like, tensor.items[1]), pred: tensor.items[0] < tensor.items[1])
@@ -51,19 +56,19 @@ module TensorStream
         when :cos
           -i_op(:sin, tensor.items[0]) * grad
         when :add
-          # rx = op(:shape, tensor.items[0])
-          # ry = op(:shape, tensor.items[1])
+          # rx = _op(:shape, tensor.items[0])
+          # ry = _op(:shape, tensor.items[1])
 
-          # ones_a = op(:ones_like, tensor.items[0])
-          # ones_b = op(:ones_like, tensor.items[1])
+          # ones_a = _op(:ones_like, tensor.items[0])
+          # ones_b = _op(:ones_like, tensor.items[1])
           # inputs = _broadcast_transform(grad * ones_a, grad2 * ones_b)
           # sx, sy = _broadcast_gradient_args(rx, ry)
 
-          # keep_dims_x = op(:rank, inputs[0]) == op(:rank, tensor.items[0])
-          # keep_dims_y = op(:rank, inputs[1]) == op(:rank, tensor.items[1])
+          # keep_dims_x = _op(:rank, inputs[0]) == _op(:rank, tensor.items[0])
+          # keep_dims_y = _op(:rank, inputs[1]) == _op(:rank, tensor.items[1])
 
-          # add_x = op(:reduce_sum, inputs[0], nil, axis: sy, keepdims: keep_dims_x)
-          # add_y = op(:reduce_sum, inputs[1], nil, axis: sx, keepdims: keep_dims_y)
+          # add_x = _op(:reduce_sum, inputs[0], nil, axis: sy, keepdims: keep_dims_x)
+          # add_y = _op(:reduce_sum, inputs[1], nil, axis: sx, keepdims: keep_dims_y)
           # _filtered_sum(add_x, add_y, wrt_dx)
           _grad_with_broadcast(tensor, wrt_dx, ->(a, b) { i_op(:add, a, b, name: 'grad_add') }, options)
         when :sub
@@ -83,15 +88,15 @@ module TensorStream
           _reduce_when_necessary(gx + gy, wrt_dx)
         when :mul
           # apply the product rule
-          rx = op(:shape, tensor.items[0])
-          ry = op(:shape, tensor.items[1])
+          rx = _op(:shape, tensor.items[0])
+          ry = _op(:shape, tensor.items[1])
           sx, sy = _broadcast_gradient_args(rx, ry)
           inputs = _broadcast_transform(tensor.items[0], tensor.items[1])
-          keep_dims_x = op(:rank, inputs[0]) == op(:rank, tensor.items[0])
-          keep_dims_y = op(:rank, inputs[1]) == op(:rank, tensor.items[1])
+          keep_dims_x = _op(:rank, inputs[0]) == _op(:rank, tensor.items[0])
+          keep_dims_y = _op(:rank, inputs[1]) == _op(:rank, tensor.items[1])
 
-          _filtered_sum(op(:reduce_sum, grad * _ds(inputs[1]), nil, axis: sy, keepdims: keep_dims_x),
-                        op(:reduce_sum, _ds(inputs[0]) * grad2, nil, axis: sx, keepdims: keep_dims_y), wrt_dx)
+          _filtered_sum(_op(:reduce_sum, grad * _ds(inputs[1]), nil, axis: sy, keepdims: keep_dims_x),
+                        _op(:reduce_sum, _ds(inputs[0]) * grad2, nil, axis: sx, keepdims: keep_dims_y), wrt_dx)
         when :reduce_mean
           input_size = i_op(:reduce_prod, i_op(:shape, tensor.items[0]))
           output_size = i_op(:reduce_prod, i_op(:shape, tensor))
@@ -100,6 +105,8 @@ module TensorStream
           (grad / i_op(:cast, factor, data_type: grad.dtype))
         when :reduce_sum
           grad
+        when :reciprocal
+          -grad * (i_cons(1, constant_options_1) / _ds(tensor.items[0])**2)
         when :stop_gradient
           return i_cons(0, constant_options)
         when :matmul
@@ -113,20 +120,20 @@ module TensorStream
           identity_1 = i_op(:ones, [s0[0], s1[1]], nil, data_type: tensor.items[1].data_type)
 
           matmul_da = i_op(:matmul, identity_0, tensor.items[1], transpose_b: true,
-                                                                 pad_zeros: true,
-                                                                 name:        'matrix_dx')
+                                                                pad_zeros: true,
+                                                                name:        'matrix_dx')
           matmul_db = i_op(:matmul, tensor.items[0], identity_1, transpose_a: true,
-                                                                 pad_zeros: true,
-                                                                 name:        'matrix_dy')
-          # matmul_db = op(:transpose, matmul_db, nil).first
+                                                                pad_zeros: true,
+                                                                name:        'matrix_dy')
+          # matmul_db = _op(:transpose, matmul_db, nil).first
 
-          # begin_a = op(:zeros, op(:rank, matmul_db), nil, data_type: :int32, name: 'begin_a')
-          # matmul_b_shape = op(:shape, matmul_db)
+          # begin_a = _op(:zeros, _op(:rank, matmul_db), nil, data_type: :int32, name: 'begin_a')
+          # matmul_b_shape = _op(:shape, matmul_db)
           # end_a = [matmul_b_shape[0], 1]
 
-          matmul_da = i_op(:cond, matmul_da[0], matmul_da, pred: op(:rank, derivative_a) > 0)
+          matmul_da = i_op(:cond, matmul_da[0], matmul_da, pred: _op(:rank, derivative_a) > 0)
 
-          # matmul_da = op(:cond, matmul_da[0], matmul_da, pred: op(:rank, derivative_a) > 0)
+          # matmul_da = _op(:cond, matmul_da[0], matmul_da, pred: _op(:rank, derivative_a) > 0)
           norm_a = i_op(:mul, derivative_a, matmul_da, name: 'grad_a_norm_mul_da')
           norm_b = i_op(:mul, derivative_b, matmul_db, name: 'grad_b_norm_mul_db')
 
@@ -173,23 +180,23 @@ module TensorStream
     end
 
     def self._reduce_when_necessary(tensor, wrt_dx)
-      rank = op(:rank, tensor)
-      dx_rank = op(:rank, wrt_dx)
-      reduced = op(:reduce_sum, tensor, nil, axis: 0)
-      op(:cond, ->{ reduced }, tensor, pred: rank > dx_rank)
+      rank = _op(:rank, tensor)
+      dx_rank = _op(:rank, wrt_dx)
+      reduced = _op(:reduce_sum, tensor, nil, axis: 0)
+      _op(:cond, ->{ reduced }, tensor, pred: rank > dx_rank)
     end
 
     def self._broadcast_gradient_args(input_a, input_b)
-      [op(:broadcast_gradient_args, input_a, input_b), op(:broadcast_gradient_args, input_b, input_a)]
+      [_op(:broadcast_gradient_args, input_a, input_b), _op(:broadcast_gradient_args, input_b, input_a)]
     end
 
     def self._broadcast_transform(input_a, input_b)
-      op(:broadcast_transform, input_a, input_b)
+      _op(:broadcast_transform, input_a, input_b)
     end
 
     # filter out zero arrays
     def self._filtered_sum(input_a, input_b, wrt_dx)
-      zero_vect = op(:zeros_like, wrt_dx)
+      zero_vect = _op(:zeros_like, wrt_dx)
       (i_op(:cond, input_a, zero_vect, pred: i_op(:reduce_sum, input_a) != 0) + i_op(:cond, input_b, zero_vect, pred: i_op(:reduce_sum, input_b) != 0))
     end
   end