tensor_stream 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7efc04eb9823375da0fefce5086213e445c11410964c067ed09adcbddad089d2
-  data.tar.gz: 63edff2f84664c2f86850a41270fb2711e65b8a6d1ebd9f88754b72280c5661a
+  metadata.gz: a3ecf49d32d385479e8dbf1c4e0ab57f120f3f0b1d78ab201d5032c42e5686f2
+  data.tar.gz: 8a299cbb8d49bac037d827f986478338cad7c3390897d3974557f4b65711da5b
 SHA512:
-  metadata.gz: 0f80c668f87e41343dc1091379246816989dc5ffbe51e2141f5c35fb021b1f3872ef52bf9bd8d911ced3bd576883401904c250efa2a6aa2e47e60150f7c7821b
-  data.tar.gz: 1bc34e5d8e82a4a9ed325a8fd7f57cc146cdff8879a84cf0d93cde61e574b14641d352c278bf020bfa1a1296af024526027b47b3c30184cb8bef37856d762ca3
+  metadata.gz: 37ed2c4b56bf7e859dfd458b87d93acdd1c0dec1079add7a7bee23493d679c2310c55a404f31ad55b64e4e7b2a6d0c46336b715752bc067242c3a496fcd7af70
+  data.tar.gz: be270b79c342832484726f83660e4b391328f41cbf61c4dd0d6f5e0016a040d1f3beac0916f7998b536b5f21336c46c187914b4fa1021c194c0f56fc4f83a4fc

data/USAGE_GUIDE.md

@@ -223,6 +223,74 @@ vars = graph.get_collection(TensorStream::GraphKeys::GLOBAL_VARIABLES)
 => [Variable(Variable:0 shape: TensorShape([]) data_type: float32)]
 ```
 
+High Performance Computing
+--------------------------
+
+TensorStream has been designed from the ground up to support multiple execution backends.
+
+What this means is you can build your models once and then be able to execute them later on specialized hardware when available like GPUs.
+
+An OpenCL backend is available that you can use for compute intensive taks like machine learning, especially those that use convolutional networks.
+
+Using OpenCL is as simple as installing the tensorstream-opencl gem
+
+```
+gem install tensor_stream-opencl
+```
+
+You can then require the library in your programs and it will get used automatically (assuming you also installed OpenCL drivers for your system)
+
+```ruby
+require 'tensor_stream'
+
+# enable OpenCL
+require 'tensor_stream/opencl'
+
+tf = TensorStream
+
+srand(5)
+seed = 5
+tf.set_random_seed(seed)
+
+SHAPES = [32, 32]
+tf = TensorStream
+sess = tf.session
+large_tensor = tf.constant(sess.run(tf.random_uniform([256, 256])))
+
+sum_axis_1 = tf.reduce_sum(large_tensor, 1)
+sess.run(sum_axis_1)
+```
+
+Using OpenCL can improve performance dramatically in scenarios involving large tensors:
+
+```
+Linux 4.15.0-46-generic #49-Ubuntu SMP
+model name	: AMD Ryzen 3 1300X Quad-Core Processor
+OpenCL device NVIDIA CUDA GeForce GTX 1060 6GB
+ruby 2.6.2p47 (2019-03-13 revision 67232) [x86_64-linux]
+
+                                           user     system      total        real
+pure ruby softmax        :             0.024724   0.000000   0.024724 (  0.024731)
+opencl    softmax        :             0.006237   0.003945   0.010182 (  0.009005)
+pure ruby matmul         :             0.679538   0.000000   0.679538 (  0.680048)
+opencl    matmul         :             0.003456   0.007965   0.011421 (  0.008568)
+pure ruby sum            :             3.210619   0.000000   3.210619 (  3.210064)
+opencl sum               :             0.002431   0.008030   0.010461 (  0.007522)
+pure ruby sum axis 1     :             3.208789   0.000000   3.208789 (  3.208125)
+opencl sum axis 1        :             0.006075   0.003963   0.010038 (  0.007679)
+pure ruby conv2d_backprop      :       3.738167   0.000000   3.738167 (  3.737946)
+opencl conv2d_backprop         :       0.031267   0.003958   0.035225 (  0.030381)
+pure ruby conv2d      :                0.794182   0.000000   0.794182 (  0.794100)
+opencl conv2d         :                0.015865   0.004020   0.019885 (  0.016878)
+```
+
+A quick glance shows not a marginal increase but an order of magnitude performance increase in most operations.
+In fact we are looking at almost a 200x faster compute on operations like matmul and softmax (essential operations in machine learning). This is not a surprise because of the "embarrasingly" parallel nature of machine learning computation. Because of this, GPUs are basically a requirement in most machine learning tasks.
+
+The code containing these benchmarks can be found at:
+
+tensor_stream-opencl/benchmark/benchmark.rb
+
 Limitations
 -----------
 

data/lib/tensor_stream/evaluator/ruby/math_ops.rb

@@ -37,6 +37,24 @@ module TensorStream
           end
         end
 
+        register_op :bias_add do |_context, _tensor, inputs|
+          value, bias = inputs
+          arr = value.flatten.each_slice(bias.size).map do |slice|
+            slice.each_with_index.map { |elem, index| elem + bias[index] }
+          end
+          TensorShape.reshape(arr, shape_eval(value))
+        end
+
+        register_op :bias_add_grad do |_context, _tensor, inputs|
+          received_grad = inputs[0]
+          bias_size = shape_eval(received_grad).last
+          grad_sum = Array.new(bias_size) { 0.0 }
+          received_grad.flatten.each_slice(bias_size) do |slice|
+            slice.each_with_index.map { |elem, index| grad_sum[index] += elem }
+          end
+          grad_sum
+        end
+
         register_op :sub, no_eval: true do |context, tensor, inputs|
           a, b = inputs
           call_vector_op(tensor, :sub, a, b, context) { |t, u| t - u }

data/lib/tensor_stream/generated_stub/ops.rb

@@ -9,12 +9,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def add(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:add, input_a, input_b, name: name)
@@ -25,14 +25,14 @@ module TensorStream
     # Returns the index with the largest value across axes of a tensor.
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type NUMERIC_TYPES)
-    # +axis+:: Describes which axis of the input tensor to reduce across. For vectors, use axis = 0 (of type INTEGER_TYPES)
+    # @param input_a tensor X (of type NUMERIC_TYPES)
+    # @param axis Describes which axis of the input tensor to reduce across. For vectors, use axis = 0 (of type INTEGER_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
-    # +:dimension+:: Same as axis
-    # +:output_type+:: Output data type defaults to int32 default (:int32)
+    # @option name Optional name
+    # @option dimension Same as axis
+    # @option output_type Output data type defaults to int32 default (:int32)
+    # @return Tensor
     def argmax(input_a, axis = nil, name: nil, dimension: nil, output_type: :int32)
       check_allowed_types(input_a, TensorStream::Ops::NUMERIC_TYPES)
       check_allowed_types(axis, TensorStream::Ops::INTEGER_TYPES)
@@ -44,14 +44,14 @@ module TensorStream
     # Returns the index with the smallest value across axes of a tensor.
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type NUMERIC_TYPES)
-    # +axis+:: Describes which axis of the input tensor to reduce across. For vectors, use axis = 0 (of type INTEGER_TYPES)
+    # @param input_a tensor X (of type NUMERIC_TYPES)
+    # @param axis Describes which axis of the input tensor to reduce across. For vectors, use axis = 0 (of type INTEGER_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
-    # +:dimension+:: Same as axis
-    # +:output_type+:: Output data type defaults to int32 default (:int32)
+    # @option name Optional name
+    # @option dimension Same as axis
+    # @option output_type Output data type defaults to int32 default (:int32)
+    # @return Tensor
     def argmin(input_a, axis = nil, name: nil, dimension: nil, output_type: :int32)
       check_allowed_types(input_a, TensorStream::Ops::NUMERIC_TYPES)
       check_allowed_types(axis, TensorStream::Ops::INTEGER_TYPES)
@@ -63,11 +63,11 @@ module TensorStream
     # Returns element-wise smallest integer in not less than x
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type FLOATING_POINT_TYPES)
+    # @param input_a tensor X (of type FLOATING_POINT_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def ceil(input_a, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
       _op(:ceil, input_a, name: name)
@@ -78,11 +78,11 @@ module TensorStream
     # Computes cos of input element-wise.
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type FLOATING_POINT_TYPES)
+    # @param input_a tensor X (of type FLOATING_POINT_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def cos(input_a, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
       _op(:cos, input_a, name: name)
@@ -94,12 +94,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def div(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:div, input_a, input_b, name: name)
@@ -111,12 +111,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def equal(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:equal, input_a, input_b, name: name)
@@ -129,12 +129,12 @@ module TensorStream
     # dimension index axis starts at zero; if you specify a negative number for axis it is counted backward from the end.
     #
     #
-    # Params:
-    # +input+:: A tensor
-    # +axis+:: Specifies the dimension index at which to expand the shape of input. Must be in the range [-rank(input) - 1, rank(input)].
+    # @param input A tensor
+    # @param axis Specifies the dimension index at which to expand the shape of input. Must be in the range [-rank(input) - 1, rank(input)].
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def expand_dims(input, axis, name: nil)
       _op(:expand_dims, input, axis, name: name)
     end
@@ -144,12 +144,12 @@ module TensorStream
     # This operation creates a tensor of shape dims and fills it with value.
     #
     #
-    # Params:
-    # +dims+:: tensor shape
-    # +value+:: scalar value to fill with
+    # @param dims tensor shape
+    # @param value scalar value to fill with
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def fill(dims, value, name: nil)
       _op(:fill, dims, value, name: name)
     end
@@ -159,11 +159,11 @@ module TensorStream
     # Returns element-wise largest integer not greater than x.
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type FLOATING_POINT_TYPES)
+    # @param input_a tensor X (of type FLOATING_POINT_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def floor(input_a, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
       _op(:floor, input_a, name: name)
@@ -175,12 +175,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def floor_div(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:floor_div, input_a, input_b, name: name)
@@ -192,12 +192,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def greater(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:greater, input_a, input_b, name: name)
@@ -209,29 +209,46 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def greater_equal(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:greater_equal, input_a, input_b, name: name)
     end
 
 
+    ##
+    # Returns the truth value of (x < y) element-wise.
+    #
+    # This operation supports broadcasting
+    #
+    # @param input_a tensor X
+    # @param input_b tensor Y
+    #
+    # Options:
+    # @option name Optional name
+    # @return Tensor
+    def less(input_a, input_b, name: nil)
+      input_a, input_b = apply_data_type_coercion(input_a, input_b)
+      _op(:less, input_a, input_b, name: name)
+    end
+
+
     ##
     # Returns the truth value of (x <= y) element-wise.
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def less_equal(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:less_equal, input_a, input_b, name: name)
@@ -242,11 +259,11 @@ module TensorStream
     # Computes natural logarithm of x element-wise.
     #
     #
-    # Params:
-    # +input+:: tensor X
+    # @param input tensor X
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def log(input, name: nil)
       _op(:log, input, name: name)
     end
@@ -257,14 +274,14 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:transpose_a+:: Transpose matrix A first default (false)
-    # +:transpose_b+:: Transpose matrix B first default (false)
-    # +:name+:: Optional name
+    # @option transpose_a Transpose matrix A first default (false)
+    # @option transpose_b Transpose matrix B first default (false)
+    # @option name Optional name
+    # @return Tensor
     def mat_mul(input_a, input_b, transpose_a: false, transpose_b: false, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:mat_mul, input_a, input_b, transpose_a: transpose_a, transpose_b: transpose_b, name: name)
@@ -277,12 +294,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X (of type NUMERIC_TYPES)
-    # +input_b+:: tensor Y (of type NUMERIC_TYPES)
+    # @param input_a tensor X (of type NUMERIC_TYPES)
+    # @param input_b tensor Y (of type NUMERIC_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def max(input_a, input_b, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::NUMERIC_TYPES)
       check_allowed_types(input_b, TensorStream::Ops::NUMERIC_TYPES)
@@ -296,12 +313,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X (of type NUMERIC_TYPES)
-    # +input_b+:: tensor Y (of type NUMERIC_TYPES)
+    # @param input_a tensor X (of type NUMERIC_TYPES)
+    # @param input_b tensor Y (of type NUMERIC_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def min(input_a, input_b, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::NUMERIC_TYPES)
       check_allowed_types(input_b, TensorStream::Ops::NUMERIC_TYPES)
@@ -315,12 +332,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def mod(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:mod, input_a, input_b, name: name)
@@ -332,12 +349,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def mul(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:mul, input_a, input_b, name: name)
@@ -348,16 +365,33 @@ module TensorStream
     # Computes numerical negative value element-wise.
     #
     #
-    # Params:
-    # +input+:: tensor X
+    # @param input tensor X
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def negate(input, name: nil)
       _op(:negate, input, name: name)
     end
 
 
+    ##
+    # Returns the truth value of (x != y) element-wise.
+    #
+    # This operation supports broadcasting
+    #
+    # @param input_a tensor X
+    # @param input_b tensor Y
+    #
+    # Options:
+    # @option name Optional name
+    # @return Tensor
+    def not_equal(input_a, input_b, name: nil)
+      input_a, input_b = apply_data_type_coercion(input_a, input_b)
+      _op(:not_equal, input_a, input_b, name: name)
+    end
+
+
     ##
     # Creates a tensor with all elements set to 1.
     # Given a single tensor (tensor), this operation returns a
@@ -365,12 +399,12 @@ module TensorStream
     # Optionally, you can specify a new type (dtype) for the returned tensor.
     #
     #
-    # Params:
-    # +input+:: A tensor
+    # @param input A tensor
     #
     # Options:
-    # +:dtype+:: Optional new data type to cast into
-    # +:name+:: Optional name
+    # @option dtype Optional new data type to cast into
+    # @option name Optional name
+    # @return Tensor
     def ones_like(input, dtype: nil, name: nil)
       _op(:ones_like, input, data_type: dtype, name: name)
     end
@@ -381,12 +415,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def pow(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:pow, input_a, input_b, name: name)
@@ -401,13 +435,13 @@ module TensorStream
     # If axis has no entries, all dimensions are reduced, and a tensor with a single element is returned.
     #
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +axis+:: tensor X (of type INTEGER_TYPES)
+    # @param input_a tensor X
+    # @param axis tensor X (of type INTEGER_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
-    # +:keepdims+:: If true, retains reduced dimensions with length 1. default (false)
+    # @option name Optional name
+    # @option keepdims If true, retains reduced dimensions with length 1. default (false)
+    # @return Tensor
     def prod(input_a, axis = nil, name: nil, keepdims: false)
       check_allowed_types(axis, TensorStream::Ops::INTEGER_TYPES)
       input_a = TensorStream.convert_to_tensor(input_a)
@@ -422,15 +456,15 @@ module TensorStream
     # Outputs random values from a uniform distribution.
     #
     #
-    # Params:
-    # +shape+:: A 1-D integer Tensor or array. The shape of the output tensor.
+    # @param shape A 1-D integer Tensor or array. The shape of the output tensor.
     #
     # Options:
-    # +:name+:: Optional name
-    # +:dtype+:: The type of the output: float16, float32, float64, int32, or int64 default (:float32)
-    # +:minval+:: A 0-D Tensor or ruby value of type dtype. The lower bound on the range of random values to generate. Defaults to 0. default (0)
-    # +:maxval+:: A 0-D Tensor or ruby value of type dtype. The upper bound on the range of random values to generate. Defaults to 1 if dtype is floating point. default (1)
-    # +:seed+::  A ruby integer. Used to create a random seed for the distribution. See set_random_seed for behavior.
+    # @option name Optional name
+    # @option dtype The type of the output: float16, float32, float64, int32, or int64 default (:float32)
+    # @option minval A 0-D Tensor or ruby value of type dtype. The lower bound on the range of random values to generate. Defaults to 0. default (0)
+    # @option maxval A 0-D Tensor or ruby value of type dtype. The upper bound on the range of random values to generate. Defaults to 1 if dtype is floating point. default (1)
+    # @option seed  A ruby integer. Used to create a random seed for the distribution. See set_random_seed for behavior.
+    # @return Tensor
     def random_uniform(shape, name: nil, dtype: :float32, minval: 0, maxval: 1, seed: nil)
       _op(:random_uniform, shape, name: name, dtype: dtype, minval: minval, maxval: maxval, seed: seed)
     end
@@ -441,15 +475,15 @@ module TensorStream
     # Creates a sequence of numbers that begins at start and extends by increments of delta up to but not including limit.
     #
     #
-    # Params:
-    # +start+:: Acts as first entry in the range if limit is not nil; otherwise, acts as range limit and first entry defaults to 0.
-    # +limit+:: Upper limit of sequence, exclusive. If nil, defaults to the value of start while the first entry of the range defaults to 0.
-    # +delta+:: Number that increments start. Defaults to 1.
+    # @param start Acts as first entry in the range if limit is not nil; otherwise, acts as range limit and first entry defaults to 0.
+    # @param limit Upper limit of sequence, exclusive. If nil, defaults to the value of start while the first entry of the range defaults to 0.
+    # @param delta Number that increments start. Defaults to 1.
     #
     # Options:
-    # +:name+::  A name for the operation. Defaults to "range". default ("range")
-    # +:dtype+:: The type of the elements of the resulting tensor.
-    # +:output_type+:: Output data type defaults to int32 default (:int32)
+    # @option name  A name for the operation. Defaults to "range". default ("range")
+    # @option dtype The type of the elements of the resulting tensor.
+    # @option output_type Output data type defaults to int32 default (:int32)
+    # @return Tensor
     def range(start = 0, limit = 0, delta = 1, name: "range", dtype: nil, output_type: :int32)
       _op(:range, start, limit, delta, name: name, dtype: dtype, output_type: output_type)
     end
@@ -459,11 +493,11 @@ module TensorStream
     # Returns the rank of a tensor
     #
     #
-    # Params:
-    # +input+:: A tensor
+    # @param input A tensor
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def rank(input, name: nil)
       input = convert_to_tensor(input)
       return cons(input.shape.ndims) if input.shape.known?
@@ -476,12 +510,12 @@ module TensorStream
     # Given tensor, this operation returns a tensor that has the same values as tensor with shape shape.
     #
     #
-    # Params:
-    # +input+:: A tensor
-    # +shape+:: A new tensor shape
+    # @param input A tensor
+    # @param shape A new tensor shape
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def reshape(input, shape, name: nil)
       _op(:reshape, input, shape, name: name)
     end
@@ -491,11 +525,11 @@ module TensorStream
     # Rounds the values of a tensor to the nearest integer, element-wise
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type FLOATING_POINT_TYPES)
+    # @param input_a tensor X (of type FLOATING_POINT_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def round(input_a, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
       _op(:round, input_a, name: name)
@@ -506,12 +540,12 @@ module TensorStream
     # This operation returns a 1-D integer tensor representing the shape of input
     #
     #
-    # Params:
-    # +input+:: A tensor
+    # @param input A tensor
     #
     # Options:
-    # +:name+:: Optional name
-    # +:out_type+:: Optional output type default (:int32)
+    # @option name Optional name
+    # @option out_type Optional output type default (:int32)
+    # @return Tensor
     def shape(input, name: nil, out_type: :int32)
       return constant(shape_eval(input, out_type), dtype: out_type, name: "Shape/#{name}") if input.is_a?(Array) && !input[0].is_a?(Tensor)
       return constant(input.shape.shape, dtype: out_type, name: "Shape/#{input.name}_c") if shape_full_specified(input)
@@ -523,11 +557,11 @@ module TensorStream
     # Computes sigmoid of x element-wise.
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type FLOATING_POINT_TYPES)
+    # @param input_a tensor X (of type FLOATING_POINT_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def sigmoid(input_a, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
       _op(:sigmoid, input_a, name: name)
@@ -540,11 +574,11 @@ module TensorStream
     # Zero is returned for NaN inputs.
     #
     #
-    # Params:
-    # +input_a+:: tensor X
+    # @param input_a tensor X
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def sign(input_a, name: nil)
       _op(:sign, input_a, name: name)
     end
@@ -554,11 +588,11 @@ module TensorStream
     # Computes sin of input element-wise.
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type FLOATING_POINT_TYPES)
+    # @param input_a tensor X (of type FLOATING_POINT_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def sin(input_a, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
       _op(:sin, input_a, name: name)
@@ -570,12 +604,12 @@ module TensorStream
     # Returns a 0-D Tensor representing the number of elements in input of type out_type. Defaults to :int32.
     #
     #
-    # Params:
-    # +input+:: A tensor
+    # @param input A tensor
     #
     # Options:
-    # +:name+:: Optional name
-    # +:out_type+:: Optional output type default (:int32)
+    # @option name Optional name
+    # @option out_type Optional output type default (:int32)
+    # @return Tensor
     def size(input, name: nil, out_type: :int32)
       _op(:size, input, name: name, out_type: out_type)
     end
@@ -586,12 +620,12 @@ module TensorStream
     #
     # This operation supports broadcasting
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +input_b+:: tensor Y
+    # @param input_a tensor X
+    # @param input_b tensor Y
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def sub(input_a, input_b, name: nil)
       input_a, input_b = apply_data_type_coercion(input_a, input_b)
       _op(:sub, input_a, input_b, name: name)
@@ -607,13 +641,13 @@ module TensorStream
     # If axis has no entries, all dimensions are reduced, and a tensor with a single element is returned.
     #
     #
-    # Params:
-    # +input_a+:: tensor X
-    # +axis+:: tensor X (of type INTEGER_TYPES)
+    # @param input_a tensor X
+    # @param axis tensor X (of type INTEGER_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
-    # +:keepdims+:: If true, retains reduced dimensions with length 1. default (false)
+    # @option name Optional name
+    # @option keepdims If true, retains reduced dimensions with length 1. default (false)
+    # @return Tensor
     def sum(input_a, axis = nil, name: nil, keepdims: false)
       check_allowed_types(axis, TensorStream::Ops::INTEGER_TYPES)
       input_a = TensorStream.convert_to_tensor(input_a)
@@ -628,11 +662,11 @@ module TensorStream
     # Computes tan of input element-wise.
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type FLOATING_POINT_TYPES)
+    # @param input_a tensor X (of type FLOATING_POINT_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def tan(input_a, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
       _op(:tan, input_a, name: name)
@@ -643,11 +677,11 @@ module TensorStream
     # Computes tanh of input element-wise.
     #
     #
-    # Params:
-    # +input_a+:: tensor X (of type FLOATING_POINT_TYPES)
+    # @param input_a tensor X (of type FLOATING_POINT_TYPES)
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def tanh(input_a, name: nil)
       check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
       _op(:tanh, input_a, name: name)
@@ -661,12 +695,12 @@ module TensorStream
     # and the values of input are replicated multiples[i] times along the 'i'th dimension. For example, tiling [a b c d] by [2] produces [a b c d a b c d].
     #
     #
-    # Params:
-    # +input+:: A tensor
-    # +multiples+:: Must be one of the following types: int32, int64. 1-D. Length must be the same as the number of dimensions in input
+    # @param input A tensor
+    # @param multiples Must be one of the following types: int32, int64. 1-D. Length must be the same as the number of dimensions in input
     #
     # Options:
-    # +:name+:: Optional name
+    # @option name Optional name
+    # @return Tensor
     def tile(input, multiples, name: nil)
       _op(:tile, input, multiples, name: name)
     end
@@ -676,12 +710,12 @@ module TensorStream
     # Creates a tensor with all elements set to zero
     #
     #
-    # Params:
-    # +shape+:: A 1-D integer Tensor or ruby array. The shape of the output tensor.
+    # @param shape A 1-D integer Tensor or ruby array. The shape of the output tensor.
     #
     # Options:
-    # +:dtype+:: Optional name default (:float32)
-    # +:name+:: Optional name
+    # @option dtype Optional name default (:float32)
+    # @option name Optional name
+    # @return Tensor
     def zeros(shape, dtype: :float32, name: nil)
       _op(:zeros, shape, dtype: dtype, name: name)
     end

data/lib/tensor_stream/generated_stub/stub_file.erb

@@ -9,12 +9,12 @@ module TensorStream
 <%end%>    #
     #<% if op.supports_broadcasting? %> This operation supports broadcasting
     #<% end %>
-    # Params:
-<% op.parameters.each do |param| %>    # +<%= param[:name] %>+:: <%= param[:description]%><%if param[:validate]%> (of type <%= param[:validate] %>)<%end%>
+<% op.parameters.each do |param| %>    # @param <%= param[:name] %> <%= param[:description]%><%if param[:validate]%> (of type <%= param[:validate] %>)<%end%>
 <% end %>    #
     # Options:
-<% op.options.each do |k, v| %>    # +:<%= k %>+:: <%= v[:description]%><% if v[:default_value] != :nil %> default (<%= v[:default_value] %>)<%end%>
-<%end%>    def <%= op.operation.to_s %>(<%= (op.expand_params(true) + op.expand_options(true)).join(', ') %>)
+<% op.options.each do |k, v| %>    # @option <%= k %> <%= v[:description]%><% if v[:default_value] != :nil %> default (<%= v[:default_value] %>)<%end%>
+<%end%>    # @return Tensor
+    def <%= op.operation.to_s %>(<%= (op.expand_params(true) + op.expand_options(true)).join(', ') %>)
 <%= op.generate_body %>
     end
 <% op.aliases.each do |a|%>

data/lib/tensor_stream/nn/nn_ops.rb

@@ -137,6 +137,19 @@ module TensorStream
       def conv2d(input, filter, strides, padding, name: nil)
         _op(:conv2d, input, filter, strides: strides, padding: padding, name: name)
       end
+
+      ##
+      # Adds bias to value.
+      #
+      # This is a narrow version of tf add where the bias is restructed to 1-D only
+      def bias_add(value, bias, data_format: nil, name: nil)
+        value = TensorStream.convert_to_tensor(value, name: "input")
+        bias = TensorStream.convert_to_tensor(bias, dtype: value.dtype, name: "bias")
+
+        raise TensorStreamError, "value must be at least rank 2" if value.shape.known? && value.shape.ndims < 2
+
+        _op(:bias_add, value, bias, data_format: data_format, name: name)
+      end
     end
   end
 

data/lib/tensor_stream/op_maker.rb

@@ -2,7 +2,8 @@ class TensorStream::OpMaker
   attr_reader :operation, :description, :parameters,
               :options, :gradient, :check_types,
               :supports_broadcast, :data_type_coercion,
-              :aliases, :custom, :infer_type_proc, :exclude
+              :aliases, :custom, :infer_type_proc, :exclude,
+              :data_type_block
 
   def initialize(op)
     @operation = op
@@ -58,6 +59,22 @@ class TensorStream::OpMaker
     context_caller.instance_exec(tensor, &@ops[tensor.operation].infer_type_proc)
   end
 
+  def self.infer_data_type(context_caller, tensor, passed_data_type)
+    return passed_data_type if passed_data_type
+
+    if @ops[tensor.operation] && @ops[tensor.operation].data_type_block
+      context_caller.instance_exec(tensor, &@ops[tensor.operation].data_type_block)
+    else
+      if tensor.inputs[0]
+        tensor.inputs[0].data_type
+      elsif tensor.inputs[1]
+        tensor.inputs[1].data_type
+      else
+        :unknown
+      end
+    end
+  end
+
   def self.each_op(&block)
     @ops.values.sort_by { |op| op.operation }.reject(&:exclude).each do |op|
       block.call(op)
@@ -122,6 +139,10 @@ class TensorStream::OpMaker
     @infer_type_proc = block
   end
 
+  def define_data_type(&block)
+    @data_type_block = block
+  end
+
   def expand_params(print_defaults)
     @parameters.map { |param|
       print_defaults && param[:default_value] ? "#{param[:name]} = #{default_with_nil(param[:default_value])}" : "#{param[:name]}"

data/lib/tensor_stream/operation.rb

@@ -96,7 +96,7 @@ module TensorStream
         options[:data_type]
       when :fill
         @inputs[1].data_type
-      when :greater, :less, :equal, :not_equal, :greater_equal, :less_equal, :logical_and
+      when :logical_and
         :boolean
       when :shape, :rank, :shape_n
         options[:out_type] || :int32
@@ -119,15 +119,7 @@ module TensorStream
           @inputs[0].data_type
         end
       else
-        return passed_data_type if passed_data_type
-
-        if @inputs[0]
-          @inputs[0].data_type
-        elsif @inputs[1]
-          @inputs[1].data_type
-        else
-          :unknown
-        end
+        OpMaker.infer_data_type(self, self, passed_data_type)
       end
     end
 

data/lib/tensor_stream/ops.rb

@@ -163,14 +163,6 @@ module TensorStream
       _op(:ones, shape, data_type: dtype, name: name)
     end
 
-    ##
-    # Returns the truth value of (x < y) element-wise.
-    # This operation supports broadcasting
-    def less(input_a, input_b, name: nil)
-      check_data_types(input_a, input_b)
-      _op(:less, input_a, input_b, name: name)
-    end
-
     ##
     # Returns the truth value of x AND y element-wise.
     def logical_and(input_a, input_b, name: nil)

data/lib/tensor_stream/ops/bias_add.rb

@@ -0,0 +1,16 @@
+TensorStream::OpMaker.define_operation :bias_add do |op|
+  op.what_it_does "Adds bias to value."
+
+  op.parameter :value, "A Tensor", :nil, validate: 'NUMERIC_TYPES'
+  op.parameter :bias, "A 1 D tensor", :nil, validate: 'NUMERIC_TYPES'
+
+  op.supports_broadcasting!
+  op.exclude!
+
+  op.option :name, "Optional name", :nil
+  op.option :data_format, "A string. 'NHWC' and 'NCHW' are supported.", :nil
+
+  op.define_gradient do |grad, node, _params|
+    [grad, _op(:bias_add_grad, grad, data_format: node.options[:data_format])]
+  end
+end

data/lib/tensor_stream/ops/equal.rb

@@ -12,4 +12,8 @@ TensorStream::OpMaker.define_operation :equal do |op|
   op.define_gradient do |grad, node, params|
     _min_or_max_grad(node.inputs, grad, ->(a, b) { ts.equal(a, b) })
   end
+
+  op.define_data_type do
+    :boolean
+  end
 end

data/lib/tensor_stream/ops/greater.rb

@@ -8,4 +8,8 @@ TensorStream::OpMaker.define_operation :greater do |op|
   op.supports_broadcasting!
 
   op.option :name, "Optional name", :nil
+
+  op.define_data_type do
+    :boolean
+  end
 end

data/lib/tensor_stream/ops/greater_equal.rb

@@ -8,4 +8,8 @@ TensorStream::OpMaker.define_operation :greater_equal do |op|
   op.supports_broadcasting!
 
   op.option :name, "Optional name", :nil
+
+  op.define_data_type do
+    :boolean
+  end
 end

data/lib/tensor_stream/ops/less.rb

@@ -0,0 +1,19 @@
+TensorStream::OpMaker.define_operation :less do |op|
+  op.what_it_does "Returns the truth value of (x < y) element-wise."
+
+  op.parameter :input_a, "tensor X"
+  op.parameter :input_b, "tensor Y"
+
+  op.apply_data_type_coercion!
+  op.supports_broadcasting!
+
+  op.option :name, "Optional name", :nil
+
+  op.define_gradient do |grad, node, _params|
+    _min_or_max_grad(node.inputs, grad, ->(a, b) { ts.less(a, b) })
+  end
+
+  op.define_data_type do
+    :boolean
+  end
+end

data/lib/tensor_stream/ops/less_equal.rb

@@ -12,4 +12,8 @@ TensorStream::OpMaker.define_operation :less_equal do |op|
   op.define_gradient do |grad, node, params|
     _min_or_max_grad(node.inputs, grad, ->(a, b) { ts.greater_equal(a, b) })
   end
+
+  op.define_data_type do
+    :boolean
+  end
 end

data/lib/tensor_stream/ops/not_equal.rb

@@ -0,0 +1,19 @@
+TensorStream::OpMaker.define_operation :not_equal do |op|
+  op.what_it_does "Returns the truth value of (x != y) element-wise."
+
+  op.parameter :input_a, "tensor X"
+  op.parameter :input_b, "tensor Y"
+
+  op.apply_data_type_coercion!
+  op.supports_broadcasting!
+
+  op.option :name, "Optional name", :nil
+
+  op.define_gradient do |grad, node, params|
+    _min_or_max_grad(node.inputs, grad, ->(a, b) { ts.not_equal(a, b) })
+  end
+
+  op.define_data_type do
+    :boolean
+  end
+end

data/lib/tensor_stream/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
-  VERSION = "1.0.5".freeze
+  VERSION = "1.0.6".freeze
 
   def self.version
     VERSION

data/samples/neural_networks/lstm.rb

@@ -0,0 +1,22 @@
+# A ruby port of the example code discussed by Martin Gorner in
+# "TensorFlow and Deep Learning without a PhD, Part 1 (Google Cloud Next '17)""
+#
+# https://www.youtube.com/watch?v=u4alGiomYP4
+#
+# Requirements:
+#   mnist-learn gem
+#   opencl_ruby_ffi gem
+require "bundler/setup"
+require "tensor_stream"
+require "mnist-learn"
+
+# Enable OpenCL hardware accelerated computation, not using OpenCL can be very slow
+# gem install tensor_stream-opencl
+require 'tensor_stream/opencl'
+
+tf = TensorStream
+
+# Import MNIST data
+puts "downloading minst data"
+mnist = Mnist.read_data_sets("/tmp/data", one_hot: true)
+puts "downloading finished"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tensor_stream
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
 platform: ruby
 authors:
 - Joseph Emmanuel Dayo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-20 00:00:00.000000000 Z
+date: 2019-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -317,6 +317,7 @@ files:
 - lib/tensor_stream/ops/add.rb
 - lib/tensor_stream/ops/argmax.rb
 - lib/tensor_stream/ops/argmin.rb
+- lib/tensor_stream/ops/bias_add.rb
 - lib/tensor_stream/ops/case.rb
 - lib/tensor_stream/ops/cast.rb
 - lib/tensor_stream/ops/ceil.rb
@@ -330,6 +331,7 @@ files:
 - lib/tensor_stream/ops/floor_div.rb
 - lib/tensor_stream/ops/greater.rb
 - lib/tensor_stream/ops/greater_equal.rb
+- lib/tensor_stream/ops/less.rb
 - lib/tensor_stream/ops/less_equal.rb
 - lib/tensor_stream/ops/log.rb
 - lib/tensor_stream/ops/mat_mul.rb
@@ -338,6 +340,7 @@ files:
 - lib/tensor_stream/ops/mod.rb
 - lib/tensor_stream/ops/mul.rb
 - lib/tensor_stream/ops/negate.rb
+- lib/tensor_stream/ops/not_equal.rb
 - lib/tensor_stream/ops/ones_like.rb
 - lib/tensor_stream/ops/pow.rb
 - lib/tensor_stream/ops/prod.rb
@@ -384,6 +387,7 @@ files:
 - samples/datasets/iris.data
 - samples/jupyter_notebooks/linear_regression.ipynb
 - samples/neural_networks/iris.rb
+- samples/neural_networks/lstm.rb
 - samples/neural_networks/mnist_data.rb
 - samples/neural_networks/raw_neural_net_sample.rb
 - samples/neural_networks/rnn.rb