tensor_stream 1.0.4 → 1.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5d2223b3321554a5529fc7be45f6be485f66d0024d8b3ff4fadb8adfb20ba2a
-  data.tar.gz: 229532234896767058fc8d3dff0897650504345aaed1ce3d57b13d09f017f19e
+  metadata.gz: 8f7d54f45a96ee2ed86af5916339747701171476e2b1cd6197f4f78d3f7f2eb3
+  data.tar.gz: f8a1c615ebf5f67de35e0e6ac84ac531fc5306b62787ac0d2d952f474eb97bad
 SHA512:
-  metadata.gz: '084f3d8fa7e74fccdbcbc42e004716520e9dc7a0cf7568ecbab8182e21ea0930ee58da9ca262b245e28e38df99e20f7dca029007c4e11030a3f6d62b4dd2e089'
-  data.tar.gz: a0c9e9987f557ed3cb301114c8f7bb6a91e1344dfeb96d72fbb4c61d850568e1d73ae8b060cabb655954a1d7b36379579eb221abfb955204454e28eae94cdd03
+  metadata.gz: 4607a3c117c98f21594bcbf12b98a1e927dab88c2847d4516c4f3dd3502b821e8d4b2b8e17e085c3ed122eec1e339baced66e3822fed2c5c6e3c8e10d0121e08
+  data.tar.gz: dd2f7b6c971a25b90a4404319231de6386aef0b536a1df9eca84a3a881e6e9e2faa903fd7fb72b9a05ba8588ec5ffdd153be0072e6c850960fff9ecaecf7b6bc

data/.gitignore

@@ -7,6 +7,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/embeddings/
 *.gem
 samples/.ipynb_checkpoints/
 

data/CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.7] - 2019-04-08
+- [NEW]    - Support for nn.embedding_lookup
+- [NEW]    - l2_normalize, dynamic_partition
+- [NEW OP] - New Ops: rsqrt, top_k, strided_slice
+- [NEW]    - Support for ranges in tensors (e.g. t[0...2] via strided slice)
+- [SAMPLES] - Add samples for handling word vectors
+
+## [1.0.5] - 2019-03-20
+- [BUG FIX] - Fix not wrapping a stack op on some arrays. Should fix rnn sample
+ 
 ## [0.9.10] - 2019-01-02
 - [BUG FIX] - remove pry-byebug include (Thanks @samgooi4189)
 - Update Changelog for 0.9.9
@@ -19,7 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [NEW OP] Convolutional networks - conv2d, conv2d_backprop_filter, conv2d_backprop_input
 - [IMAGE] Exposed image resampling options
 - [BUG FIX] fix argmin, argmax handling of NaN values
-
+ 
 ## [0.9.5] - 2018-11-05
 - [NEW OP] assert_equal, relu6
 - [TRAINING] learning_rate_decay, dropout
@@ -134,4 +144,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - reworked auto differentiation, fix a number of bugs related to auto differentiation, smaller derivative programs
 - alpha support for saving to pbtext format, added graphml generation
 - significant number of ops added
-- ops that support broadcasting now work better
+- ops that support broadcasting now work better

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM circleci/ruby:2.4.1-node-browsers
+FROM circleci/ruby:2.6.1-node-browsers
 RUN sudo apt-get update -q && sudo apt-get install --no-install-recommends -yq alien wget unzip clinfo \
     && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 RUN export DEVEL_URL="https://software.intel.com/file/531197/download" \

data/USAGE_GUIDE.md

@@ -223,6 +223,74 @@ vars = graph.get_collection(TensorStream::GraphKeys::GLOBAL_VARIABLES)
 => [Variable(Variable:0 shape: TensorShape([]) data_type: float32)]
 ```
 
+High Performance Computing
+--------------------------
+
+TensorStream has been designed from the ground up to support multiple execution backends.
+
+What this means is you can build your models once and then be able to execute them later on specialized hardware when available like GPUs.
+
+An OpenCL backend is available that you can use for compute intensive taks like machine learning, especially those that use convolutional networks.
+
+Using OpenCL is as simple as installing the tensorstream-opencl gem
+
+```
+gem install tensor_stream-opencl
+```
+
+You can then require the library in your programs and it will get used automatically (assuming you also installed OpenCL drivers for your system)
+
+```ruby
+require 'tensor_stream'
+
+# enable OpenCL
+require 'tensor_stream/opencl'
+
+tf = TensorStream
+
+srand(5)
+seed = 5
+tf.set_random_seed(seed)
+
+SHAPES = [32, 32]
+tf = TensorStream
+sess = tf.session
+large_tensor = tf.constant(sess.run(tf.random_uniform([256, 256])))
+
+sum_axis_1 = tf.reduce_sum(large_tensor, 1)
+sess.run(sum_axis_1)
+```
+
+Using OpenCL can improve performance dramatically in scenarios involving large tensors:
+
+```
+Linux 4.15.0-46-generic #49-Ubuntu SMP
+model name	: AMD Ryzen 3 1300X Quad-Core Processor
+OpenCL device NVIDIA CUDA GeForce GTX 1060 6GB
+ruby 2.6.2p47 (2019-03-13 revision 67232) [x86_64-linux]
+
+                                           user     system      total        real
+pure ruby softmax        :             0.024724   0.000000   0.024724 (  0.024731)
+opencl    softmax        :             0.006237   0.003945   0.010182 (  0.009005)
+pure ruby matmul         :             0.679538   0.000000   0.679538 (  0.680048)
+opencl    matmul         :             0.003456   0.007965   0.011421 (  0.008568)
+pure ruby sum            :             3.210619   0.000000   3.210619 (  3.210064)
+opencl sum               :             0.002431   0.008030   0.010461 (  0.007522)
+pure ruby sum axis 1     :             3.208789   0.000000   3.208789 (  3.208125)
+opencl sum axis 1        :             0.006075   0.003963   0.010038 (  0.007679)
+pure ruby conv2d_backprop      :       3.738167   0.000000   3.738167 (  3.737946)
+opencl conv2d_backprop         :       0.031267   0.003958   0.035225 (  0.030381)
+pure ruby conv2d      :                0.794182   0.000000   0.794182 (  0.794100)
+opencl conv2d         :                0.015865   0.004020   0.019885 (  0.016878)
+```
+
+A quick glance shows not a marginal increase but an order of magnitude performance increase in most operations.
+In fact we are looking at almost a 200x faster compute on operations like matmul and softmax (essential operations in machine learning). This is not a surprise because of the "embarrasingly" parallel nature of machine learning computation. Because of this, GPUs are basically a requirement in most machine learning tasks.
+
+The code containing these benchmarks can be found at:
+
+tensor_stream-opencl/benchmark/benchmark.rb
+
 Limitations
 -----------
 

data/lib/tensor_stream.rb

@@ -23,6 +23,7 @@ require "tensor_stream/operation"
 require "tensor_stream/placeholder"
 require "tensor_stream/control_flow"
 require "tensor_stream/dynamic_stitch"
+require "tensor_stream/math/math_ops"
 require "tensor_stream/nn/nn_ops"
 require "tensor_stream/evaluator/evaluator"
 require "tensor_stream/graph_serializers/packer"

data/lib/tensor_stream/evaluator/base_evaluator.rb

@@ -2,11 +2,17 @@ module TensorStream
   # Evaluator base module
   module Evaluator
     class OutputGroup
+      include Enumerable
+
       attr_accessor :outputs, :data_types
       def initialize(outputs = [], data_types = [])
         @outputs = outputs
         @data_types = data_types
       end
+
+      def each
+        @outputs.map { |output| yield output }
+      end
     end
 
     class UnsupportedOp < RuntimeError
@@ -131,7 +137,7 @@ module TensorStream
           time.to_i * (10**9) + time.nsec
         end
 
-        instance_exec(execution_context, tensor, resolved_inputs, &op[:block]).tap do
+        instance_exec(execution_context, tensor, resolved_inputs, &op[:block]).tap do |result|
           if profile_enabled?
             time = Time.now
             end_time = time.to_i * (10**9) + time.nsec
@@ -222,11 +228,25 @@ module TensorStream
 
     def self.register_evaluator(klass, name, index = 0)
       @evaluators ||= {}
+      @storage_managers ||= {}
       @evaluators[name] = {name: name, class: klass, index: index}
+      @storage_managers[klass] = klass.get_storage_manager
     end
 
     def self.default_evaluators
       evaluators.values.sort { |v| v[:index] }.reverse.map { |v| v[:class] }
     end
+
+    def self.clear_storages(graph)
+      @storage_managers.values.each { |manager| manager.clear_variables(graph) }
+    end
+
+    def self.read_variable(graph, name)
+      @storage_managers.values.each do |manager|
+        return manager.read_value(graph, name) if manager.exists?(graph, name)
+      end
+
+      nil
+    end
   end
 end

data/lib/tensor_stream/evaluator/evaluator.rb

@@ -1,5 +1,6 @@
 require "tensor_stream/evaluator/ruby_evaluator"
 require "tensor_stream/evaluator/buffer"
+require "tensor_stream/evaluator/evaluator_utils"
 
 module TensorStream
   module Evaluator

data/lib/tensor_stream/evaluator/evaluator_utils.rb

@@ -0,0 +1,20 @@
+module TensorStream
+  class EvaluatorUtils
+    extend TensorStream::StringHelper
+
+    def self.get_evaluator_classes(evaluators)
+      @evaluator_classes ||= if evaluators.is_a?(Array)
+        if evaluators.empty?
+          TensorStream::Evaluator.default_evaluators
+        else
+          evaluators.collect { |name| Object.const_get("TensorStream::Evaluator::#{camelize(name.to_s)}") }
+        end
+      elsif evaluators.nil?
+        TensorStream::Evaluator.default_evaluators
+      else
+        [Object.const_get("TensorStream::Evaluator::#{camelize(evaluators.to_s)}")]
+      end
+      @evaluator_classes
+    end
+  end
+end

data/lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb

@@ -30,6 +30,16 @@ module TensorStream
       end
     end
 
+    def array_set!(input, value)
+        input.each_with_index do |element, index|
+          if element.is_a?(Array)
+            array_set(element, value) 
+          else
+            input[index] = value[index]
+          end
+        end
+    end
+
     def truncate(input, target_shape)
       rank = get_rank(input)
       return input if rank.zero?
@@ -331,5 +341,55 @@ module TensorStream
         value.nil? ? arr : value
       end
     end
+
+    def strided_slice(value, slices = [])
+      current_slice = slices.dup
+      selection = current_slice.shift
+      return value if selection.nil?
+
+      b, e, stride = selection
+
+      b = value.size + b if b < 0
+      e = value.size + e + 1 if e < 0
+
+      indexes = if stride < 0
+                  b.downto(e).select.with_index { |elem, index| (index % stride.abs) == 0 }
+                else
+                  (b...e).step(stride)
+                end
+
+      indexes.map do |index|
+        strided_slice(value[index], current_slice)
+      end
+    end
+
+    def strided_slice_grad(value, grad, x, slices)
+      current_slice = slices.dup
+      selection = current_slice.shift
+      current_shape = x.shift
+
+      if selection.nil?
+        array_set!(value, grad)
+      end
+
+      b, e, stride = selection
+
+      b = value.size + b if b < 0
+      e = value.size + e + 1 if e < 0
+
+      indexes = if stride < 0
+                  b.downto(e).select.with_index { |elem, index| (index % stride.abs) == 0 }
+                else
+                  (b...e).step(stride)
+                end
+
+      indexes.each_with_index do |index, grad_index|
+        if (value[index].is_a?(Array))
+          strided_slice_grad(value[index], grad[grad_index], x.dup, current_slice.dup)
+        else
+          value[index] = grad[grad_index]
+        end
+      end
+    end
   end
 end

data/lib/tensor_stream/evaluator/ruby/array_ops.rb

@@ -22,8 +22,9 @@ module TensorStream
           merged
         end
 
-        register_op :gather do |_context, _tensor, inputs|
+        register_op :gather do |_context, tensor, inputs|
           params, indexes = inputs
+          raise "axis !=0 not supported" if tensor.options[:axis] != 0
           gather(params, indexes)
         end
 
@@ -216,7 +217,14 @@ module TensorStream
 
         register_op :range do |_context, _tensor, inputs|
           start, limit, delta = inputs
+
           raise " delta !=0 " if delta.zero?
+
+          if limit.zero?
+            limit = start
+            start = 0
+          end
+
           raise " Requires start <= limit when delta > 0" if (start > limit) && delta > 0
           raise " Requires start >= limit when delta < 0" if (start < limit) && delta < 0
 
@@ -399,6 +407,50 @@ module TensorStream
           end
         end
 
+        register_op :dynamic_partition do |context, tensor, inputs|
+          data, partitions = inputs
+          num_partitions = tensor.options[:num_partitions]
+          output_arr = Array.new(num_partitions) { [] }
+
+          partitions.each_with_index do |part, index|
+            output_arr[part] << data[index]
+          end
+          TensorStream::Evaluator::OutputGroup.new(output_arr, num_partitions.times.map { tensor.data_type })
+        end
+
+        register_op :gather_grad do |context, tensor, inputs|
+          grad, indexes, input_shape = inputs
+          output = Array.new(input_shape.reduce(:*)) { fp_type?(tensor.data_type) ? 0.0 : 0 }
+          indexes.each_with_index.map do |x, index|
+            output[x] += grad[index]
+          end
+          TensorShape.reshape(output, input_shape)
+        end
+
+        register_op :strided_slice do |_context, _tensor, inputs|
+          value, b_index, e_index, stride = inputs
+          slices = b_index.zip(e_index).zip(stride).map do |params|
+            selection, stride = params
+            s, e = selection
+            [s, e, stride]
+          end
+          strided_slice(value, slices)
+        end
+
+        register_op :strided_slice_grad do |_context, tensor, inputs|
+          x, b_index, e_index, stride, grad = inputs
+          slices = b_index.zip(e_index).zip(stride).map do |params|
+            selection, stride = params
+            s, e = selection
+            [s, e, stride]
+          end
+
+          target_val = generate_vector(x, generator: ->() { fp_type?(tensor.data_type) ? 0.0 : 0 })
+
+          strided_slice_grad(target_val, grad, x.dup, slices.dup)
+          target_val
+        end
+
         def merge_dynamic_stitch(merged, indexes, data, context)
           indexes.each_with_index do |ind, m|
             if ind.is_a?(Array)

data/lib/tensor_stream/evaluator/ruby/images_ops.rb

@@ -1,5 +1,6 @@
 require "chunky_png"
 
+
 module TensorStream
   module ImagesOps
     def self.included(klass)
@@ -49,6 +50,31 @@ module TensorStream
           TensorShape.reshape(image_data, [image.height, image.width, channels])
         end
 
+        register_op :decode_jpg do |_context, tensor, inputs|
+          require "jpeg"
+
+          content = inputs[0]
+          channels = tensor.options[:channels]
+          channels = 3 if channels.zero?
+
+          image = Jpeg::Image.open_buffer(content)
+          source_channels = image.color_info == :gray ? 1 : 3
+
+          image_data = image.raw_data.map do |pixel|
+            if source_channels == channels
+              pixel
+            elsif source_channels = 1 && channels == 3
+              [pixel, pixel, pixel]
+            elsif source_channels = 3 && channels == 1
+              raise TensorStream::ValueError, "color to grayscale not supported for jpg"
+            end
+          end.flatten
+
+          image_data.map!(&:to_f) if fp_type?(tensor.data_type)
+
+          TensorShape.reshape(image_data, [image.height, image.width, channels])
+        end
+
         register_op :encode_png do |_context, tensor, inputs|
           image_data = inputs[0]
           height, width, channels = shape_eval(image_data)

data/lib/tensor_stream/evaluator/ruby/math_ops.rb

@@ -37,6 +37,24 @@ module TensorStream
           end
         end
 
+        register_op :bias_add do |_context, _tensor, inputs|
+          value, bias = inputs
+          arr = value.flatten.each_slice(bias.size).map do |slice|
+            slice.each_with_index.map { |elem, index| elem + bias[index] }
+          end
+          TensorShape.reshape(arr, shape_eval(value))
+        end
+
+        register_op :bias_add_grad do |_context, _tensor, inputs|
+          received_grad = inputs[0]
+          bias_size = shape_eval(received_grad).last
+          grad_sum = Array.new(bias_size) { 0.0 }
+          received_grad.flatten.each_slice(bias_size) do |slice|
+            slice.each_with_index.map { |elem, index| grad_sum[index] += elem }
+          end
+          grad_sum
+        end
+
         register_op :sub, no_eval: true do |context, tensor, inputs|
           a, b = inputs
           call_vector_op(tensor, :sub, a, b, context) { |t, u| t - u }
@@ -111,6 +129,15 @@ module TensorStream
           call_op(inputs[0], context) { |t, _b| Math.sqrt(t) }
         end
 
+        register_op :rsqrt, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context) { |t, _b|  1 / Math.sqrt(t) }
+        end
+
+        register_op :rsqrt_grad, no_eval: true do |context, tensor, inputs|
+          y, grad = inputs
+          call_vector_op(tensor, :rsqrt_grad, y, grad, context) { |_y, g| 0.5 * g * (_y ** 3) }
+        end
+
         register_op :floor, no_eval: true do |context, _tensor, inputs|
           call_op(inputs[0], context) { |t, _b| t.floor }
         end
@@ -135,6 +162,25 @@ module TensorStream
           call_op(inputs[0], context) { |t, _b| 1 - Math.tanh(t) * Math.tanh(t) }
         end
 
+        register_op :top_k do |context, tensor, inputs|
+          values, k = inputs
+          v_shape = shape_eval(values)
+
+          sorted = tensor.options[:sorted]
+          work_values = TensorShape.reshape(values, [-1, v_shape.last])
+          work_values.map! do |row|
+            last_k = row.map.with_index { |r, index| [r, index] }.sort! { |a,b| a[0] <=> b[0] }.last(k)
+            last_k.reverse! if sorted
+            last_k
+          end
+
+          top_k = work_values.map { |row| row.map { |r| r[0] } }
+          top_indices = work_values.map { |row| row.map { |r| r[1] } }
+          v_shape[-1] = k
+
+          TensorStream::Evaluator::OutputGroup.new([TensorShape.reshape(top_k, v_shape), TensorShape.reshape(top_indices, v_shape)], [tensor.inputs[0].data_type, :int32])
+        end
+
         register_op(%i[argmax arg_max]) do |_context, tensor, inputs|
           axis = inputs[1] || 0
           rank = get_rank(inputs[0])
@@ -241,13 +287,22 @@ module TensorStream
           raise "#{tensor.inputs[0].name} rank must be greater than 1" if rank_a < 2
           raise "#{tensor.inputs[1].name} rank must be greater than 1" if rank_b < 2
 
-          matrix_a = matrix_a.transpose if tensor.options[:transpose_a]
-          matrix_b = matrix_b.transpose if tensor.options[:transpose_b]
-
           # check matrix dimensions
-          raise TensorStream::ValueError, "incompatible shape sizes for matrix multiplication (#{matrix_a[0].size} != #{matrix_b.size}) #{shape_eval(matrix_a)} vs #{shape_eval(matrix_b)}" if matrix_a[0].size != matrix_b.size
+          if rank_a >= 3
+            matrix_a.zip(matrix_b).map do |m_a, m_b|
+              matmul(m_a, m_b, tensor)
+            end
+          else
+            matmul(matrix_a, matrix_b, tensor)
+          end
+        end
+
+        def matmul(m_a, m_b, tensor)
+          m_a = m_a.transpose if tensor.options[:transpose_a]
+          m_b = m_b.transpose if tensor.options[:transpose_b]
+          raise TensorStream::ValueError, "incompatible shape sizes for matrix multiplication (#{m_a[0].size} != #{m_b.size}) #{shape_eval(m_a)} vs #{shape_eval(m_b)}" if m_a[0].size != m_b.size
 
-          (Matrix[*matrix_a] * Matrix[*matrix_b]).to_a
+          (Matrix[*m_a] * Matrix[*m_b]).to_a
         end
 
         register_op %i[max maximum], noop: true do |context, tensor, inputs|