tensor_stream 0.9.8 → 0.9.9

data/lib/tensor_stream/placeholder.rb

@@ -11,13 +11,13 @@ module TensorStream
       @is_const = false
 
       @name = [@graph.get_name_scope, options[:name] || build_name].compact.reject(&:empty?).join('/')
-      @graph.add_node(self)
+      @op = Graph.get_default_graph.add_op!(:placeholder, data_type: @data_type, shape: @shape, internal_name: @name)
     end
 
     private
 
     def build_name
-      "Placeholder#{graph.get_placeholder_counter}:#{@rank}"
+      "Placeholder#{graph.get_placeholder_counter}"
     end
   end
 end

data/lib/tensor_stream/session.rb

@@ -56,8 +56,20 @@ module TensorStream
 
       # scan for placeholders and assign value
       if options[:feed_dict]
-        options[:feed_dict].keys.each do |k|
-          context[k.name.to_sym] = options[:feed_dict][k] if k.is_a?(Placeholder)
+        options[:feed_dict].each_key do |k|
+          if k.is_a?(Placeholder)
+            context[k.name.to_sym] = options[:feed_dict][k]
+          elsif k.is_a?(String)
+            target_graph = args[0].graph
+            node = target_graph.get_node(k)
+            if node.operation == :placeholder
+              context[k.to_sym] = options[:feed_dict][k]
+            else
+              raise "Cannot find placeholder with the name of #{k}"
+            end
+          else
+            raise "Invalid placeholder type passed key must be a string or a placeholder type"
+          end
         end
       end
 
@@ -125,7 +137,7 @@ module TensorStream
           end
         end
       else
-        run_with_session_context(tensor_arr, session_context, context)
+        run_with_session_context(tensor_arr.op, session_context, context)
       end
     end
 

data/lib/tensor_stream/tensor.rb

@@ -4,40 +4,11 @@ module TensorStream
   # Base class that defines a tensor like interface
   class Tensor
     include OpHelper
-    attr_reader :graph
-    attr_accessor :name, :data_type, :shape, :rank, :native_buffer, :is_const,
-                  :value, :breakpoint, :internal, :source, :given_name,
-                  :consumers, :outputs, :device
-
-    def initialize(data_type, rank, shape, options = {})
-      setup_initial_state(options)
-      @data_type = data_type
-      @rank = rank
-      @breakpoint = false
-      @shape = TensorShape.new(shape, rank)
-      @value = nil
-
-      @is_const = options[:const] || false
-      @internal = options[:internal]
-      @name = [@graph.get_name_scope, options[:name] || build_name].compact.reject(&:empty?).join('/')
-      @given_name = @name
-
-      if options[:value]
-        if options[:value].is_a?(Array)
-          # check if single dimenstion array is passed
-          options[:value] = reshape(options[:value], shape.reverse.dup) if shape.size >= 2 && !options[:value].empty? && !options[:value][0].is_a?(Array)
-
-          @value = options[:value].map { |v| v.is_a?(Tensor) ? Tensor.cast_dtype(v, @data_type) : v }
-        elsif !shape.empty?
-          @value = reshape(Tensor.cast_dtype(options[:value], @data_type), shape.dup)
-        else
-          @value = Tensor.cast_dtype(options[:value], @data_type)
-        end
-        @shape = TensorShape.new(shape_eval(@value))
-      end
+    include TensorMixins
 
-      @graph.add_node(self)
-    end
+    attr_reader :graph, :value
+    attr_accessor :name, :data_type, :shape, :rank, :native_buffer, :is_const,
+                  :internal, :source, :given_name, :outputs, :op
 
     def internal?
       !!@internal
@@ -47,120 +18,18 @@ module TensorStream
       @data_type
     end
 
+    def consumers
+      op.consumers
+    end
+
     def self.reset_counters
       @const_counter = 0
       @var_counter = 0
       @placeholder_counter = 0
     end
 
-    def +(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:add, self, other)
-    end
-
-    def [](index)
-      _op(:index, self, index)
-    end
-
-    def *(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:mul, self, TensorStream.convert_to_tensor(other, dtype: data_type))
-    end
-
-    def **(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:pow, self, TensorStream.convert_to_tensor(other, dtype: data_type))
-    end
-
-    def /(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:div, self, TensorStream.convert_to_tensor(other, dtype: data_type))
-    end
-
-    def -(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:sub, self, TensorStream.convert_to_tensor(other, dtype: data_type))
-    end
-
-    def -@
-      _op(:negate, self, nil)
-    end
-
-    def %(other)
-      TensorStream.mod(self, other)
-    end
-
-    def floor
-      TensorStream.floor(self)
-    end
-
-    def ceil
-      TensorStream.ceil(self)
-    end
-
-    def zero?
-      _op(:equal, self, TensorStream.constant(0, dtype: data_type, name: 'equal/is_zero?'))
-    end
-
-    def ==(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:equal, self, other)
-    end
-
-    def <(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:less, self, other)
-    end
-
-    def !=(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:not_equal, self, other)
-    end
-
-    def >(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:greater, self, other)
-    end
-
-    def >=(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:greater_equal, self, other)
-    end
-
-    def <=(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:less_equal, self, other)
-    end
-
-    def and(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:logical_and, self, other)
-    end
-
-    def matmul(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:mat_mul, self, other)
-    end
-
-    def dot(other)
-      _a, other = TensorStream.check_data_types(self, other)
-      _op(:mat_mul, self, other)
-    end
-
-    ##
-    # Apply a reduction to tensor
-    def reduce(op_type)
-      reduce_op = case op_type.to_sym
-                  when :+
-                    :sum
-                  when :*
-                    :prod
-                  else
-                    raise "unsupported reduce op type #{op_type}"
-                  end
-      raise "blocks are not supported for tensors" if block_given?
-
-      _op(reduce_op, self, nil)
+    def device
+      @op.device
     end
 
     def collect(&block)
@@ -171,27 +40,12 @@ module TensorStream
       @name
     end
 
-    def const_value
-      return nil unless is_const
-
-      @value
-    end
-
-    def op
-      @op ||= is_const ? _op(:const, self, nil, name: name) : _op(:variable, self, nil, name: name)
-    end
-
     def eval(options = {})
       Session.default_session.run(self, options)
     end
 
     def to_h
       {
-        name: @name,
-        value: hashify_tensor(@value),
-        dtype: @data_type,
-        shape: @shape,
-        const: !!is_const,
       }
     end
 
@@ -295,19 +149,6 @@ module TensorStream
       @source = format_source(caller_locations)
     end
 
-    def add_consumer(consumer)
-      @consumers ||= Set.new
-      @consumers << consumer.name if consumer.name != name
-    end
-
-    def setup_output(consumer)
-      @outputs << consumer.name unless @outputs.include?(consumer.name)
-    end
-
-    def propagate_consumer(consumer)
-      add_consumer(consumer)
-    end
-
     def propagate_outputs
       # nop
     end
@@ -322,20 +163,22 @@ module TensorStream
       end
     end
 
-    def reshape(arr, shape)
+    def _reshape(arr, shape)
       if arr.is_a?(Array)
         return arr if shape.size < 2
+
         slice = shape.shift
         arr.each_slice(slice).collect do |s|
-          reshape(s, shape)
+          _reshape(s, shape)
         end
       else
         return arr if shape.empty?
+
         slice = shape.shift
         return arr if slice.nil?
 
         Array.new(slice) do
-          reshape(arr, shape.dup)
+          _reshape(arr, shape.dup)
         end
       end
     end

data/lib/tensor_stream/tensor_shape.rb

@@ -9,6 +9,8 @@ module TensorStream
     end
 
     def to_s
+      return "" if @shape.nil?
+
       dimensions = @shape.collect do |r|
         "Dimension(#{r})"
       end.join(',')
@@ -24,7 +26,7 @@ module TensorStream
     end
 
     def scalar?
-      shape.size.zero?
+      known? && shape.size.zero?
     end
 
     def known?

data/lib/tensor_stream/train/saver.rb

@@ -7,7 +7,7 @@ module TensorStream
     class Saver
       include TensorStream::OpHelper
 
-      def save(session, outputfile, global_step: nil,
+      def save(session, outputdir, global_step: nil,
                latest_filename: nil,
                meta_graph_suffix: 'meta',
                write_meta_graph: true,
@@ -33,20 +33,22 @@ module TensorStream
           }
         end
 
-        basename = File.basename(outputfile)
-        path = File.dirname(outputfile)
-
-        new_filename = File.join(path, [basename, gs].compact.join('-'))
+        FileUtils.mkdir_p(outputdir)
+        basename = 'model'
+        File.write(File.join(outputdir, "#{basename}.meta"), { "gs" => gs }.to_json)
+        new_filename = File.join(outputdir, [basename, gs, '.ckpt'].compact.join('-'))
         File.write(new_filename, output_dump.to_yaml)
         if write_meta_graph
-          graph_filename = "#{basename}.pbtext"
-          TensorStream.train.write_graph(graph, path, graph_filename)
+          graph_filename = "#{basename}.yaml"
+          TensorStream.train.write_graph(graph, outputdir, graph_filename, serializer: :yaml)
         end
-        path
+        outputdir
       end
 
-      def restore(_session, inputfile)
-        input_dump = YAML.safe_load(File.read(inputfile))
+      def restore(_session, modelpath)
+        meta_data = JSON.parse(File.read(File.join(modelpath, "model.meta")))
+        gs = meta_data['gs']
+        input_dump = YAML.safe_load(File.read(File.join(modelpath, ['model', gs, '.ckpt'].compact.join('-'))))
 
         vars = TensorStream::Graph.get_default_graph.get_collection(GraphKeys::GLOBAL_VARIABLES)
         vars.each do |variable|

data/lib/tensor_stream/trainer.rb

@@ -13,12 +13,17 @@ module TensorStream
   module Trainer
     extend TensorStream::Train::Utils
     extend TensorStream::Train::LearningRateDecay
+    extend TensorStream::StringHelper
 
-    def self.write_graph(graph, path, filename, as_text: true, serializer: TensorStream::Pbtext)
+    def self.write_graph(graph, path, filename, as_text: true, serializer: :yaml)
       raise "only supports as_text=true for now" unless as_text
 
+      serializer = constantize("TensorStream::#{camelize(serializer.to_s)}") if serializer.is_a?(Symbol)
+
       new_filename = File.join(path, filename)
-      File.write(new_filename, serializer.new.get_string(graph))
+      serializer.new.get_string(graph).tap do |str|
+        File.write(new_filename, str)
+      end
     end
   end
 end

data/lib/tensor_stream/utils.rb

@@ -45,7 +45,7 @@ module TensorStream
     # Creates a variable
     # A variable maintains state across sessions
     def variable(value, name: nil, initializer: nil, graph: nil, dtype: nil, trainable: true)
-      op = Operation.new(:assign, nil, value)
+      op = Graph.get_default_graph.add_op(:assign, nil, value)
       common_options = {
         initializer: initializer || op,
         name: name,
@@ -54,15 +54,16 @@ module TensorStream
         trainable: trainable
       }
       tensor = if value.is_a?(String)
-                 TensorStream::Variable.new(dtype || :string, 0, [], get_variable_scope, common_options)
+                 i_var(dtype || :string, 0, [], get_variable_scope, common_options)
                elsif value.is_a?(Integer)
-                 TensorStream::Variable.new(dtype || :int32, 0, [], get_variable_scope, common_options)
+                 i_var(dtype || :int32, 0, [], get_variable_scope, common_options)
                elsif value.is_a?(Float)
-                 TensorStream::Variable.new(dtype || :float32, 0, [], get_variable_scope, common_options)
+                 i_var(dtype || :float32, 0, [], get_variable_scope, common_options)
                else
-                 TensorStream::Variable.new(dtype || :float32, 0, nil, get_variable_scope, common_options)
+                 i_var(dtype || :float32, 0, nil, get_variable_scope, common_options)
                end
-      op.inputs[0] = tensor
+      op.set_input(0, tensor.op)
+      Graph.get_default_graph.add_node(op)
       tensor
     end
 
@@ -163,13 +164,13 @@ module TensorStream
       shared_options = { const: true, value: value, name: name, internal: internal }
 
       if value.is_a?(Float)
-        TensorStream::Tensor.new(dtype || :float32, 0, shape || [], shared_options)
+        TensorStream::Constant.new(dtype || :float32, 0, shape || [], shared_options)
       elsif value.is_a?(Integer)
-        TensorStream::Tensor.new(dtype || :int32, 0, shape || [], shared_options)
+        TensorStream::Constant.new(dtype || :int32, 0, shape || [], shared_options)
       elsif value.is_a?(String)
-        TensorStream::Tensor.new(dtype || :string, 0, shape || [], shared_options)
+        TensorStream::Constant.new(dtype || :string, 0, shape || [], shared_options)
       elsif !!value == value
-        TensorStream::Tensor.new(dtype || :boolean, 0, shape || [], shared_options)
+        TensorStream::Constant.new(dtype || :boolean, 0, shape || [], shared_options)
       elsif value.is_a?(Array)
         dimension = shape || shape_eval(value)
         rank = dimension.size
@@ -179,7 +180,7 @@ module TensorStream
         value = Tensor.cast_dtype(value, cur_dtype) unless dtype.nil?
 
         shared_options[:value] = value
-        TensorStream::Tensor.new(cur_dtype, rank, dimension, shared_options)
+        TensorStream::Constant.new(cur_dtype, rank, dimension, shared_options)
       end
     end
 
@@ -239,6 +240,7 @@ module TensorStream
     def convert_to_tensor(value, dtype: nil, name: nil)
       return value if value.is_a?(Tensor)
       return convert_to_tensor(value.call) if value.is_a?(Proc)
+      # raise "Invalid tensor value" if value.nil?
 
       if value.is_a?(Array) && value[0].is_a?(Tensor)
         return TensorStream.stack(value) if value.size > 1

data/lib/tensor_stream/utils/freezer.rb

@@ -0,0 +1,37 @@
+module TensorStream
+  class Freezer
+    include OpHelper
+
+    ##
+    # Utility class to convert variables to constants for production deployment
+    #
+    def convert(model_file, checkpoint_file, output_file)
+      TensorStream.graph.as_default do |current_graph|
+        YamlLoader.new.load_from_string(File.read(model_file))
+        saver = TensorStream::Train::Saver.new
+        saver.restore(nil, checkpoint_file)
+        output_buffer = TensorStream::Yaml.new.get_string(current_graph) do |graph, node_key|
+          node = graph.get_tensor_by_name(node_key)
+          if node.operation == :variable_v2
+            value = node.container
+            options = {
+              value: value,
+              data_type: node.data_type,
+              shape: shape_eval(value)
+            }
+            const_op = TensorStream::Operation.new(current_graph, inputs: [], options: options)
+            const_op.name = node.name
+            const_op.operation = :const
+            const_op.data_type = node.data_type
+            const_op.shape = TensorShape.new(shape_eval(value))
+
+            const_op
+          else
+            node
+          end
+        end
+        File.write(output_file, output_buffer)
+      end
+    end
+  end
+end