tensor_stream 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4f9e6d7a1640d4b24b92d7706bcab88ab5bb294551f38dc6ad403eb1b2a761b
-  data.tar.gz: 71c245e394be382e3976f4b2b9989c2b89538f22cc40984a5a84e07fb0bca597
+  metadata.gz: d42a81e850271f080d408c52f2bea15a07c6d41ee3c6790dc04e48f2ab485364
+  data.tar.gz: a4aedfd3c9a532f31ea195c58124644fcae143726d37daa7a4a6afc6b39f439b
 SHA512:
-  metadata.gz: a04ca2c61064dc4fb67f2653acbde962f8f8b5f56a5e5f27760f44f6032b81c61307e3b207bbb4ec42bd1713d06315ba760a710a712fb9f124efa6c7f06a7246
-  data.tar.gz: ae8e29f412c1b63b854604696df4858ce4d3fb078c982e53a0b85fd8ff6b546c2b9502ecb4d253cb6d0f9255aa1e16b1aaa445da769a0b06bb85ed55a0aebd55
+  metadata.gz: 7d9fff1a8af14878c50469cfcee7942d2800906fe388504261290958628861b29d973ea65a3cf986cd1657acf490d2bc7164ec9ec2c14dc7c5d6c25121c6737f
+  data.tar.gz: 42ef6af8fafd1a7f7f069e03f8c344bec87bd15217def07d859d4b33374a038e1b7ed54ac685b901d3ed9b51fd351300cc61553ca45f0591fc4a7c5e50bcee53

data/.rubocop.yml

@@ -6,6 +6,12 @@ AllCops:
     - tensor_stream.gemspec
     - Rakefile
 
+Style/StringLiterals:
+  Enabled: false
+
+Layout/TrailingBlankLines:
+  Enabled: false
+
 Metrics/LineLength:
   Max: 200
 
@@ -21,6 +27,10 @@ Metrics/MethodLength:
 Metrics/CyclomaticComplexity:
   Enabled: false
 
+Metrics/BlockLength:
+  Exclude:
+    - lib/tensor_stream/math_gradients.rb
+
 Naming/AccessorMethodName:
   Exclude:
     - lib/tensor_stream.rb

data/CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.0] - 2018-08-08
+### Added
+- [NEW OP] expand_dims, min, acos, asin, add_n
+- Add parse_from_string support. Parse tensorflow pbtext files into tensor_stream
+
+### Fixes
+- Tweaks to GradientDescentOptimizer to expose additional methods based on tensorflow
+
 ## [0.6.0] - 2018-07-21
 ### Added
 - [NEW OP] fill, floor_div, dynamic_stitch, mod, range, size, squared_difference

data/README.md

@@ -15,6 +15,7 @@ The goal of this gem is to have a high performance machine learning and compute
 - Provision to use your own opcode evaluator (opencl, sciruby and tensorflow backends planned)
 - Goal is to be as close to TensorFlow in behavior but with some freedom to add ruby specific enhancements (with lots of test cases)
 - eager execution (experimental)
+- (08-08-2018) Load pbtext files from tensorflow (Graph.parse_from_string)
 
 Since this is a pure ruby implementation for now, performance is not there yet. However it should be a good enough environment to learn about tensorflow and experiment with some models.
 
@@ -69,7 +70,7 @@ b = tf.variable(rand, name: "bias")
 pred = X * W + b
 
 # Mean squared error
-cost = tf.reduce_sum(tf.pow(pred - Y, 2)) / ( 2 * n_samples)
+cost = ((pred - Y) ** 2).reduce(:+) / ( 2 * n_samples)
 
 optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate).minimize(cost)
 
@@ -255,6 +256,44 @@ Note that the OpenCL evaluator provides speedup if you are using large tensors,
 
 samples/nearest_neighbor.rb contains a sample that uses opencl.
 
+## Export Import Models from tensorflow
+
+Experimental support for parsing and exporting pbtext files are supported:
+
+Exporting
+
+```ruby
+a = ts.constant([1.0, 1.0])
+b = ts.constant([1.5, 1.5])
+f = a + b
+
+File.write('my_model.pbtext', f.graph.as_graph_def)
+```
+
+Importing (Experimental)
+
+Note that not all tensorflow ops are supported, warnings will be showed
+if a certain operation is not supported yet.
+
+
+```ruby
+  pbtext = File.read(File.join('linear_regression.pbtxt'))
+
+  # create a graph from pbtext file
+  graph = TensorStream::Graph.parse_from_string(pbtext)
+
+  # reference a tensor by name from the created graph,
+  # for example you have a tensor named out
+  tensor = graph.get_tensor_by_name("out")
+
+  # set graph as default and do operations on it
+  graph.as_default do
+    sess = ts.session
+    expect(tr(sess.run(tensor))).to eq([[1.0, 1.0], [1.0, 1.0]])
+  end
+
+```
+
 # Visualization
 
 tensorstream does not support tensorboard yet, but a graphml generator is included:

data/benchmark/benchmark.rb

@@ -8,7 +8,7 @@ require 'tensor_stream/evaluator/opencl/opencl_evaluator'
 def tr(t, places = 1)
   if t.is_a?(Array)
     return t.collect do |v|
-      tr(v)
+      tr(v, places)
     end
   end
 
@@ -59,6 +59,7 @@ pow_i = tf.pow(a_int, 3)
 matmul = tf.matmul(a, b)
 out_of_order = tf.matmul(a, b) + tf.matmul(a, c)
 softmax = tf.nn.softmax(a)
+add_n = tf.add_n([a,b,c,d])
 
 puts TensorStream::Evaluator.default_evaluators
 
@@ -68,6 +69,8 @@ puts `cat /proc/cpuinfo | grep "model name" | head -1`
 device = TensorStream::Evaluator::OpenclEvaluator.default_device.native_device
 puts "OpenCL device #{device.platform.to_s} #{device.name}"
 Benchmark.bmbm do |x|
+  x.report("pure ruby add_n          :") { 100.times do sess.run(add_n) end }
+  x.report("opencl ruby add_n        :") { 100.times do sess2.run(add_n) end }
   x.report("pure ruby ooo matmul     :") { 100.times do sess.run(out_of_order) end }
   x.report("opencl    ooo matmul     :") { 100.times do sess2.run(out_of_order) end }
   x.report("pure ruby softmax        :") { 100.times do sess.run(softmax) end }

data/lib/tensor_stream.rb

@@ -2,25 +2,30 @@ require 'tensor_stream/version'
 require 'deep_merge'
 require 'matrix'
 require 'concurrent'
+require 'tensor_stream/exceptions'
 require 'tensor_stream/helpers/op_helper'
 require 'tensor_stream/helpers/string_helper'
 require 'tensor_stream/initializer'
 require 'tensor_stream/graph_keys'
 require 'tensor_stream/types'
+require 'tensor_stream/graph_builder'
 require 'tensor_stream/graph'
 require 'tensor_stream/device'
 require 'tensor_stream/session'
 require 'tensor_stream/tensor_shape'
 require 'tensor_stream/tensor'
 require 'tensor_stream/variable'
+require 'tensor_stream/variable_scope'
 require 'tensor_stream/operation'
 require 'tensor_stream/placeholder'
 require 'tensor_stream/control_flow'
 require 'tensor_stream/dynamic_stitch'
+require 'tensor_stream/train/utils'
 require 'tensor_stream/trainer'
 require 'tensor_stream/nn/nn_ops'
 require 'tensor_stream/evaluator/evaluator'
 require 'tensor_stream/graph_serializers/serializer'
+require 'tensor_stream/graph_deserializers/protobuf'
 require 'tensor_stream/graph_serializers/pbtext'
 require 'tensor_stream/graph_serializers/graphml'
 require 'tensor_stream/math_gradients'

data/lib/tensor_stream/debugging/debugging.rb

@@ -4,10 +4,12 @@ module TensorStream
 
     def add_check_numerics_ops
       graph = TensorStream.get_default_graph
-      nodes_to_process  = graph.nodes.values.select { |node| node.is_a?(Operation) }
+      nodes_to_process = graph.nodes.values.select { |node| node.is_a?(Operation) }
 
       nodes_to_process.each do |node|
-        node.inputs = node.inputs.compact.collect do |input|
+        node.inputs = node.inputs.collect do |input|
+          next if input.nil?
+
           if TensorStream::Ops::FLOATING_POINT_TYPES.include?(input.data_type)
             TensorStream.check_numerics(input, "#{node.name}/#{input.name}", name: "check/#{node.name}/#{input.name}" )
           else

data/lib/tensor_stream/device.rb

@@ -1,5 +1,6 @@
 # A tensorstream device
 module TensorStream
+  # Class that describes a supported device
   class Device
     attr_accessor :name, :type, :evaluator
     def initialize(name, type, evaluator)
@@ -8,4 +9,4 @@ module TensorStream
       @evaluator = evaluator
     end
   end
-end
+end

data/lib/tensor_stream/evaluator/base_evaluator.rb

@@ -1,5 +1,13 @@
 module TensorStream
+  # Evaluator base module
   module Evaluator
+    class OutputGroup
+      attr_accessor :outputs
+      def initialize(outputs = [])
+        @outputs = outputs
+      end
+    end
+
     class UnsupportedOp < Exception
       def initialize(tensor)
         @tensor = tensor
@@ -10,31 +18,36 @@ module TensorStream
       end
     end
 
+    # Evaluator base class
     class BaseEvaluator
-      def initialize(session, device, thread_pool: nil, log_intermediates: false)
+      def initialize(session, _device, thread_pool: nil, log_intermediates: false)
         @session = session
         @log_intermediates = log_intermediates
         @thread_pool = thread_pool || Concurrent::ImmediateExecutor.new
         @context[:compute_history] = [] if log_intermediates
       end
 
+      ##
+      # Query all supported devices
       def self.query_supported_devices
-        [Device.new("cpu", :cpu, self)]
+        [Device.new('cpu', :cpu, self)]
       end
 
       ##
       # Select the best device available in the system for this evaluator
       def self.default_device
-        Device.new("cpu", :cpu, self)
+        Device.new('cpu', :cpu, self)
       end
 
       ##
       # Selects the best device with the specified query, query can
       # be evaluator specific
-      def self.fetch_device(query = [])
-        Device.new("cpu", :cpu, self)
+      def self.fetch_device(_query = [])
+        Device.new('cpu', :cpu, self)
       end
 
+      ##
+      # Select device using uri
       def self.query_device(query)
         return default_device if query.nil? || query == :default
 
@@ -52,8 +65,8 @@ module TensorStream
 
             select_index = [devices.size - 1, select_index].min
             return devices[select_index]
-          elsif components[0] == 'cpu'
-            device_type = :cpu
+          elsif %w[cpu gpu].include?(components[0])
+            device_type = components[0].to_sym
             select_index = components[1].to_i
 
             devices = all_devices.select { |d| d.type == device_type.downcase.to_sym }
@@ -91,34 +104,32 @@ module TensorStream
 
       def invoke(tensor, execution_context)
         return eval_tensor(tensor, execution_context) unless tensor.is_a?(Operation)
+        raise UnsupportedOp.new(tensor), "op #{tensor.operation} is not yet supported" unless self.class.ops.key?(tensor.operation.to_sym)
 
-        if self.class.ops.key?(tensor.operation.to_sym)
-          op = self.class.ops[tensor.operation.to_sym]
+        op = self.class.ops[tensor.operation.to_sym]
+        op_options = op[:options]
 
-          op_options = op[:options]
-          resolved_inputs = tensor.inputs.map do |i|
-            next if i.nil?
+        resolved_inputs = tensor.inputs.map do |i|
+          next if i.nil?
 
-            if i.is_a?(Array)
-              next i.collect { |sub_item| sub_item.is_a?(Tensor) ? invoke(sub_item, execution_context) : sub_item }
-            end
+          if i.is_a?(Array)
+            next i.collect { |sub_item| sub_item.is_a?(Tensor) ? invoke(sub_item, execution_context) : sub_item }
+          end
 
-            if !op_options[:noop] && @context[:_cache][:placement][tensor.name] != @context[:_cache][:placement][i.name] # tensor is on another device or evaluator
-              cache_key = "#{tensor.graph.object_id}_#{i.name}:#{object_id}"
-              next @context[:_cache][cache_key] if @context[:_cache].key?(cache_key)
+          if !op_options[:noop] && @context[:_cache][:placement][tensor.name] != @context[:_cache][:placement][i.name] # tensor is on another device or evaluator
+            cache_key = "#{tensor.graph.object_id}_#{i.name}:#{object_id}"
+            next @context[:_cache][cache_key] if @context[:_cache].key?(cache_key)
 
-              result = @session.delegate_to_evaluator(i, @context, execution_context)
-              convert_from_buffer(i, result).tap do |buffer|
-                @context[:_cache][cache_key] = buffer if i.is_const
-              end
-            else
-              prepare_input(i, execution_context, op_options)
+            result = @session.delegate_to_evaluator(i, @context, execution_context)
+            convert_from_buffer(i, result).tap do |buffer|
+              @context[:_cache][cache_key] = buffer if i.is_const
             end
+          else
+            prepare_input(i, execution_context, op_options)
           end
-          instance_exec(execution_context, tensor, resolved_inputs, &op[:block])
-        else
-          raise UnsupportedOp.new(tensor)
         end
+
+        instance_exec(execution_context, tensor, resolved_inputs, &op[:block])
       end
 
       protected
@@ -128,13 +139,13 @@ module TensorStream
 
         input_a_args = []
         input_b_args = []
-        
-        input_a = input_b.size.times.map { |i| i < input_a.size ? input_a[i] : nil }.reverse if input_a.size < input_b.size
-        input_b = input_a.size.times.map { |i| i < input_b.size ? input_b[i] : nil }.reverse if input_a.size > input_b.size
+
+        input_a = Array.new(input_b.size) { |i| i < input_a.size ? input_a[i] : nil }.reverse if input_a.size < input_b.size
+        input_b = Array.new(input_a.size) { |i| i < input_b.size ? input_b[i] : nil }.reverse if input_a.size > input_b.size
 
         input_a.reverse.zip(input_b.reverse).each_with_index do |item, index|
           a, b = item
- 
+
           if a.nil? || b && (a < b)
             input_a_args << input_b.size - index - 1
           elsif b.nil? || a && (a > b)
@@ -142,7 +153,7 @@ module TensorStream
           end
         end
 
-        [input_a_args.reverse, input_b_args.reverse]
+       [input_a_args.reverse, input_b_args.reverse]
       end
 
       ##

data/lib/tensor_stream/evaluator/evaluator.rb

@@ -1,4 +1,3 @@
-
 require 'tensor_stream/evaluator/ruby_evaluator'
 require 'tensor_stream/evaluator/buffer'
 

data/lib/tensor_stream/evaluator/opencl/kernels/acos.cl

@@ -0,0 +1,8 @@
+% c_dtype = dtype_to_c_type(dtype)
+__kernel void acos_<%= dtype %>(const int M, const int N, __global const <%= c_dtype %> *A, __global <%= c_dtype %> *C) {
+    // Get the index of the current element to be processed
+    const int globalRow = get_global_id(0); // Row ID of C (0..M)
+    const int globalCol = get_global_id(1); // Col ID of C (0..N)
+
+    C[globalRow * N + globalCol] = acos(A[globalRow * N + globalCol]);
+}

data/lib/tensor_stream/evaluator/opencl/kernels/apply_gradient.cl

@@ -0,0 +1,9 @@
+% c_dtype = dtype_to_c_type(dtype)
+ // same dimension add floating point op
+ __kernel void apply_gradient_<%= dtype %>(const int M, const int N, __global const <%= c_dtype %> *A, __global const <%= c_dtype %> *B, __global <%= c_dtype %> *C) {
+    // Get the index of the current element to be processed
+    const int globalRow = get_global_id(0); // Row ID of C (0..M)
+    const int globalCol = get_global_id(1); // Col ID of C (0..N)
+
+    C[globalRow * N + globalCol] -= A[globalRow * N + globalCol] * B[0];
+}

data/lib/tensor_stream/evaluator/opencl/kernels/asin.cl

@@ -0,0 +1,9 @@
+
+% c_dtype = dtype_to_c_type(dtype)
+__kernel void asin_<%= dtype %>(const int M, const int N, __global const <%= c_dtype %> *A, __global <%= c_dtype %> *C) {
+    // Get the index of the current element to be processed
+    const int globalRow = get_global_id(0); // Row ID of C (0..M)
+    const int globalCol = get_global_id(1); // Col ID of C (0..N)
+
+    C[globalRow * N + globalCol] = asin(A[globalRow * N + globalCol]);
+}

data/lib/tensor_stream/evaluator/opencl/kernels/floor_mod.cl

@@ -0,0 +1,3 @@
+% c_dtype = dtype_to_c_type(dtype)
+% op = operator_to_c('mod')
+<%= render 'operand.cl', c_dtype: c_dtype, op: op, fname: 'floor_mod', dtype: "#{a}_#{b}", result_t: c_dtype %>

data/lib/tensor_stream/evaluator/opencl/kernels/log_softmax.cl

@@ -0,0 +1,26 @@
+// First naive implementation
+% c_dtype = dtype_to_c_type(dtype)
+__kernel void log_softmax_<%= dtype %>(const int N,
+                      const __global <%= c_dtype %>* A,
+                      __global <%= c_dtype %>* C) {
+
+    // Get the index of the current element to be processed
+    const int globalRow = get_global_id(0); // Row ID of C (0..M)
+
+    // Compute a single element (loop over K)
+    <%= c_dtype %> acc = 0.0f;
+    <%= c_dtype %> max = <%= min_value_for(dtype) %>;
+
+    for (int k=0; k<N; k++) {
+      max = A[globalRow*N + k] > max ? A[globalRow*N + k] : max;
+    }
+
+    for (int k=0; k<N; k++) {
+      acc += exp(A[globalRow*N + k] - max);
+    }
+
+    // Store the result
+    for (int k=0; k < N; k++) {
+      C[globalRow*N + k] = (A[globalRow*N + k] - max) - log(acc);
+    }
+}

data/lib/tensor_stream/evaluator/opencl/kernels/max.cl

@@ -5,7 +5,7 @@
     const int globalRow = get_global_id(0); // Row ID of C (0..M)
     const int globalCol = get_global_id(1); // Col ID of C (0..N)
 
-    C[globalRow * N + globalCol] = A[globalRow * N + globalCol] > B[globalRow * N + globalCol] ? A[globalRow * N + globalCol] : B[globalRow * N + globalCol];
+    C[globalRow * N + globalCol] = A[globalRow * N + globalCol] >= B[globalRow * N + globalCol] ? A[globalRow * N + globalCol] : B[globalRow * N + globalCol];
 }
 
  // 1D + Scalar floating point add op
@@ -15,9 +15,9 @@
     const int globalCol = get_global_id(1); // Col ID of C (0..N)
     
     if (switch_op == 0) {
-      C[globalRow * N + globalCol] = A[globalRow * N + globalCol] > B[0] ? A[globalRow * N + globalCol] : B[0];
+      C[globalRow * N + globalCol] = A[globalRow * N + globalCol] >= B[0] ? A[globalRow * N + globalCol] : B[0];
     } else {
-      C[globalRow * N + globalCol] = B[0] > A[globalRow * N + globalCol] ? B[0] : A[globalRow * N + globalCol];
+      C[globalRow * N + globalCol] = B[0] >= A[globalRow * N + globalCol] ? B[0] : A[globalRow * N + globalCol];
     }
 }
 
@@ -39,8 +39,8 @@
     }
 
     if (switch_op == 0) {
-      C[globalRow * N + globalCol] = A[globalRow * N + globalCol] > B[b_m_index * N2 + b_n_index] ? A[globalRow * N + globalCol] : B[b_m_index * N2 + b_n_index];
+      C[globalRow * N + globalCol] = A[globalRow * N + globalCol] >= B[b_m_index * N2 + b_n_index] ? A[globalRow * N + globalCol] : B[b_m_index * N2 + b_n_index];
     } else {
-      C[globalRow * N + globalCol] = B[b_m_index * N2 + b_n_index] > A[globalRow * N + globalCol] ? B[b_m_index * N2 + b_n_index] :  A[globalRow * N + globalCol];
+      C[globalRow * N + globalCol] = B[b_m_index * N2 + b_n_index] >= A[globalRow * N + globalCol] ? B[b_m_index * N2 + b_n_index] :  A[globalRow * N + globalCol];
     }
 }