tensor_stream 1.0.6 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3ecf49d32d385479e8dbf1c4e0ab57f120f3f0b1d78ab201d5032c42e5686f2
-  data.tar.gz: 8a299cbb8d49bac037d827f986478338cad7c3390897d3974557f4b65711da5b
+  metadata.gz: 667188a4a1ebc020c6c03c2b8530505b26e0ccb80885e03abba16c09665d6247
+  data.tar.gz: 06aebda0444eaa155d986324a0d4939bda66bbc6c6bc34762d83e46e1ba41fec
 SHA512:
-  metadata.gz: 37ed2c4b56bf7e859dfd458b87d93acdd1c0dec1079add7a7bee23493d679c2310c55a404f31ad55b64e4e7b2a6d0c46336b715752bc067242c3a496fcd7af70
-  data.tar.gz: be270b79c342832484726f83660e4b391328f41cbf61c4dd0d6f5e0016a040d1f3beac0916f7998b536b5f21336c46c187914b4fa1021c194c0f56fc4f83a4fc
+  metadata.gz: 309595e1452075516bd003a3ca99b5537430cef182000521ea951d9e51459356e236a64ea98216aeb4ea507313541ad702be6d67e3536102632af7a3ed6ca6fa
+  data.tar.gz: 194174036aaee864f96cbbd7768f5b46a91a1592b0b9dd4a167c7071c063d92f2b62db26961a00ebace8eaece74caebe9232c04b4f741293b267dcdb215928e9

data/.gitignore

@@ -7,6 +7,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/embeddings/
 *.gem
 samples/.ipynb_checkpoints/
 

data/CHANGELOG.md

@@ -4,9 +4,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.7] - 2019-04-08
+- [NEW]    - Support for nn.embedding_lookup
+- [NEW]    - l2_normalize, dynamic_partition
+- [NEW OP] - New Ops: rsqrt, top_k, strided_slice
+- [NEW]    - Support for ranges in tensors (e.g. t[0...2] via strided slice)
+- [SAMPLES] - Add samples for handling word vectors
+
 ## [1.0.5] - 2019-03-20
 - [BUG FIX] - Fix not wrapping a stack op on some arrays. Should fix rnn sample
-
+ 
 ## [0.9.10] - 2019-01-02
 - [BUG FIX] - remove pry-byebug include (Thanks @samgooi4189)
 - Update Changelog for 0.9.9
@@ -22,7 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [NEW OP] Convolutional networks - conv2d, conv2d_backprop_filter, conv2d_backprop_input
 - [IMAGE] Exposed image resampling options
 - [BUG FIX] fix argmin, argmax handling of NaN values
-
+ 
 ## [0.9.5] - 2018-11-05
 - [NEW OP] assert_equal, relu6
 - [TRAINING] learning_rate_decay, dropout
@@ -137,4 +144,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - reworked auto differentiation, fix a number of bugs related to auto differentiation, smaller derivative programs
 - alpha support for saving to pbtext format, added graphml generation
 - significant number of ops added
-- ops that support broadcasting now work better
+- ops that support broadcasting now work better

data/lib/tensor_stream.rb

@@ -23,6 +23,7 @@ require "tensor_stream/operation"
 require "tensor_stream/placeholder"
 require "tensor_stream/control_flow"
 require "tensor_stream/dynamic_stitch"
+require "tensor_stream/math/math_ops"
 require "tensor_stream/nn/nn_ops"
 require "tensor_stream/evaluator/evaluator"
 require "tensor_stream/graph_serializers/packer"

data/lib/tensor_stream/evaluator/base_evaluator.rb

@@ -2,11 +2,17 @@ module TensorStream
   # Evaluator base module
   module Evaluator
     class OutputGroup
+      include Enumerable
+
       attr_accessor :outputs, :data_types
       def initialize(outputs = [], data_types = [])
         @outputs = outputs
         @data_types = data_types
       end
+
+      def each
+        @outputs.map { |output| yield output }
+      end
     end
 
     class UnsupportedOp < RuntimeError

data/lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb

@@ -30,6 +30,16 @@ module TensorStream
       end
     end
 
+    def array_set!(input, value)
+        input.each_with_index do |element, index|
+          if element.is_a?(Array)
+            array_set(element, value) 
+          else
+            input[index] = value[index]
+          end
+        end
+    end
+
     def truncate(input, target_shape)
       rank = get_rank(input)
       return input if rank.zero?
@@ -331,5 +341,55 @@ module TensorStream
         value.nil? ? arr : value
       end
     end
+
+    def strided_slice(value, slices = [])
+      current_slice = slices.dup
+      selection = current_slice.shift
+      return value if selection.nil?
+
+      b, e, stride = selection
+
+      b = value.size + b if b < 0
+      e = value.size + e + 1 if e < 0
+
+      indexes = if stride < 0
+                  b.downto(e).select.with_index { |elem, index| (index % stride.abs) == 0 }
+                else
+                  (b...e).step(stride)
+                end
+
+      indexes.map do |index|
+        strided_slice(value[index], current_slice)
+      end
+    end
+
+    def strided_slice_grad(value, grad, x, slices)
+      current_slice = slices.dup
+      selection = current_slice.shift
+      current_shape = x.shift
+
+      if selection.nil?
+        array_set!(value, grad)
+      end
+
+      b, e, stride = selection
+
+      b = value.size + b if b < 0
+      e = value.size + e + 1 if e < 0
+
+      indexes = if stride < 0
+                  b.downto(e).select.with_index { |elem, index| (index % stride.abs) == 0 }
+                else
+                  (b...e).step(stride)
+                end
+
+      indexes.each_with_index do |index, grad_index|
+        if (value[index].is_a?(Array))
+          strided_slice_grad(value[index], grad[grad_index], x.dup, current_slice.dup)
+        else
+          value[index] = grad[grad_index]
+        end
+      end
+    end
   end
 end

data/lib/tensor_stream/evaluator/ruby/array_ops.rb

@@ -22,8 +22,9 @@ module TensorStream
           merged
         end
 
-        register_op :gather do |_context, _tensor, inputs|
+        register_op :gather do |_context, tensor, inputs|
           params, indexes = inputs
+          raise "axis !=0 not supported" if tensor.options[:axis] != 0
           gather(params, indexes)
         end
 
@@ -216,7 +217,14 @@ module TensorStream
 
         register_op :range do |_context, _tensor, inputs|
           start, limit, delta = inputs
+
           raise " delta !=0 " if delta.zero?
+
+          if limit.zero?
+            limit = start
+            start = 0
+          end
+
           raise " Requires start <= limit when delta > 0" if (start > limit) && delta > 0
           raise " Requires start >= limit when delta < 0" if (start < limit) && delta < 0
 
@@ -399,6 +407,50 @@ module TensorStream
           end
         end
 
+        register_op :dynamic_partition do |context, tensor, inputs|
+          data, partitions = inputs
+          num_partitions = tensor.options[:num_partitions]
+          output_arr = Array.new(num_partitions) { [] }
+
+          partitions.each_with_index do |part, index|
+            output_arr[part] << data[index]
+          end
+          TensorStream::Evaluator::OutputGroup.new(output_arr, num_partitions.times.map { tensor.data_type })
+        end
+
+        register_op :gather_grad do |context, tensor, inputs|
+          grad, indexes, input_shape = inputs
+          output = Array.new(input_shape.reduce(:*)) { fp_type?(tensor.data_type) ? 0.0 : 0 }
+          indexes.each_with_index.map do |x, index|
+            output[x] += grad[index]
+          end
+          TensorShape.reshape(output, input_shape)
+        end
+
+        register_op :strided_slice do |_context, _tensor, inputs|
+          value, b_index, e_index, stride = inputs
+          slices = b_index.zip(e_index).zip(stride).map do |params|
+            selection, stride = params
+            s, e = selection
+            [s, e, stride]
+          end
+          strided_slice(value, slices)
+        end
+
+        register_op :strided_slice_grad do |_context, tensor, inputs|
+          x, b_index, e_index, stride, grad = inputs
+          slices = b_index.zip(e_index).zip(stride).map do |params|
+            selection, stride = params
+            s, e = selection
+            [s, e, stride]
+          end
+
+          target_val = generate_vector(x, generator: ->() { fp_type?(tensor.data_type) ? 0.0 : 0 })
+
+          strided_slice_grad(target_val, grad, x.dup, slices.dup)
+          target_val
+        end
+
         def merge_dynamic_stitch(merged, indexes, data, context)
           indexes.each_with_index do |ind, m|
             if ind.is_a?(Array)

data/lib/tensor_stream/evaluator/ruby/math_ops.rb

@@ -129,6 +129,15 @@ module TensorStream
           call_op(inputs[0], context) { |t, _b| Math.sqrt(t) }
         end
 
+        register_op :rsqrt, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context) { |t, _b|  1 / Math.sqrt(t) }
+        end
+
+        register_op :rsqrt_grad, no_eval: true do |context, tensor, inputs|
+          y, grad = inputs
+          call_vector_op(tensor, :rsqrt_grad, y, grad, context) { |_y, g| 0.5 * g * (_y ** 3) }
+        end
+
         register_op :floor, no_eval: true do |context, _tensor, inputs|
           call_op(inputs[0], context) { |t, _b| t.floor }
         end
@@ -153,6 +162,25 @@ module TensorStream
           call_op(inputs[0], context) { |t, _b| 1 - Math.tanh(t) * Math.tanh(t) }
         end
 
+        register_op :top_k do |context, tensor, inputs|
+          values, k = inputs
+          v_shape = shape_eval(values)
+
+          sorted = tensor.options[:sorted]
+          work_values = TensorShape.reshape(values, [-1, v_shape.last])
+          work_values.map! do |row|
+            last_k = row.map.with_index { |r, index| [r, index] }.sort! { |a,b| a[0] <=> b[0] }.last(k) rescue binding.pry
+            last_k.reverse! if sorted
+            last_k
+          end
+
+          top_k = work_values.map { |row| row.map { |r| r[0] } }
+          top_indices = work_values.map { |row| row.map { |r| r[1] } }
+          v_shape[-1] = k
+
+          TensorStream::Evaluator::OutputGroup.new([TensorShape.reshape(top_k, v_shape), TensorShape.reshape(top_indices, v_shape)], [tensor.inputs[0].data_type, :int32])
+        end
+
         register_op(%i[argmax arg_max]) do |_context, tensor, inputs|
           axis = inputs[1] || 0
           rank = get_rank(inputs[0])
@@ -259,13 +287,22 @@ module TensorStream
           raise "#{tensor.inputs[0].name} rank must be greater than 1" if rank_a < 2
           raise "#{tensor.inputs[1].name} rank must be greater than 1" if rank_b < 2
 
-          matrix_a = matrix_a.transpose if tensor.options[:transpose_a]
-          matrix_b = matrix_b.transpose if tensor.options[:transpose_b]
-
           # check matrix dimensions
-          raise TensorStream::ValueError, "incompatible shape sizes for matrix multiplication (#{matrix_a[0].size} != #{matrix_b.size}) #{shape_eval(matrix_a)} vs #{shape_eval(matrix_b)}" if matrix_a[0].size != matrix_b.size
+          if rank_a >= 3
+            matrix_a.zip(matrix_b).map do |m_a, m_b|
+              matmul(m_a, m_b, tensor)
+            end
+          else
+            matmul(matrix_a, matrix_b, tensor)
+          end
+        end
+
+        def matmul(m_a, m_b, tensor)
+          m_a = m_a.transpose if tensor.options[:transpose_a]
+          m_b = m_b.transpose if tensor.options[:transpose_b]
+          raise TensorStream::ValueError, "incompatible shape sizes for matrix multiplication (#{m_a[0].size} != #{m_b.size}) #{shape_eval(m_a)} vs #{shape_eval(m_b)}" if m_a[0].size != m_b.size
 
-          (Matrix[*matrix_a] * Matrix[*matrix_b]).to_a
+          (Matrix[*m_a] * Matrix[*m_b]).to_a
         end
 
         register_op %i[max maximum], noop: true do |context, tensor, inputs|

data/lib/tensor_stream/generated_stub/ops.rb

@@ -536,6 +536,21 @@ module TensorStream
     end
 
 
+    ##
+    # Computes reciprocal of square root of x element-wise.
+    #
+    #
+    # @param input_a tensor X (of type FLOATING_POINT_TYPES)
+    #
+    # Options:
+    # @option name Optional name
+    # @return Tensor
+    def rsqrt(input_a, name: nil)
+      check_allowed_types(input_a, TensorStream::Ops::FLOATING_POINT_TYPES)
+      _op(:rsqrt, input_a, name: name)
+    end
+
+
     ##
     # This operation returns a 1-D integer tensor representing the shape of input
     #
@@ -615,6 +630,28 @@ module TensorStream
     end
 
 
+    ##
+    # Extracts a strided slice of a tensor 
+    # this op extracts a slice of size `(end-begin)/stride`
+    #   from the given `input_` tensor. Starting at the location specified by `begin`
+    #   the slice continues by adding `stride` to the index until all dimensions are
+    #   not less than `end`.
+    #   Note that a stride can be negative, which causes a reverse slice.
+    #
+    #
+    # @param input A tensor
+    # @param _begin start index
+    # @param _end end index
+    # @param strides end index
+    #
+    # Options:
+    # @option name Optional name
+    # @return Tensor
+    def strided_slice(input, _begin, _end, strides = nil, name: nil)
+      _op(:strided_slice, input, _begin, _end, strides, name: name)
+    end
+
+
     ##
     # Returns x - y element-wise.
     #
@@ -642,18 +679,20 @@ module TensorStream
     #
     #
     # @param input_a tensor X
-    # @param axis tensor X (of type INTEGER_TYPES)
+    # @param axis_p tensor X (of type INTEGER_TYPES)
     #
     # Options:
+    # @option axis axis
     # @option name Optional name
     # @option keepdims If true, retains reduced dimensions with length 1. default (false)
     # @return Tensor
-    def sum(input_a, axis = nil, name: nil, keepdims: false)
-      check_allowed_types(axis, TensorStream::Ops::INTEGER_TYPES)
+    def sum(input_a, axis_p = nil, axis: nil, name: nil, keepdims: false)
+      check_allowed_types(axis_p, TensorStream::Ops::INTEGER_TYPES)
       input_a = TensorStream.convert_to_tensor(input_a)
       return input_a if input_a.shape.scalar?
-      axis = cast_axis(input_a, axis)
-      _op(:sum, input_a, axis, name: name, keepdims: keepdims)
+      axis_p = axis_p || axis
+      axis_p = cast_axis(input_a, axis_p)
+      _op(:sum, input_a, axis_p, name: name, keepdims: keepdims)
     end
 
     alias_method :reduce_sum, :sum
@@ -706,6 +745,23 @@ module TensorStream
     end
 
 
+    ##
+    # Finds values and indices of the `k` largest entries for the last dimension.
+    #
+    #
+    # @param input 1-D or higher `Tensor` with last dimension at least `k`.
+    # @param k 0-D `int32` `Tensor`.  Number of top elements to look for along the last dimension (along each row for matrices)
+    #
+    # Options:
+    # @option sorted If true the resulting `k` elements will be sorted by the values in descending order. default (true)
+    # @option name Optional name
+    # @return Tensor
+    def top_k(input, k = 1, sorted: true, name: nil)
+      result = _op(:top_k, input, k, sorted: sorted, name: name)
+      [result[0], result[1]]
+    end
+
+
     ##
     # Creates a tensor with all elements set to zero
     #

data/lib/tensor_stream/helpers/tensor_mixins.rb

@@ -5,7 +5,16 @@ module TensorStream
     end
 
     def [](index)
-      _op(:index, self, index)
+      if index.is_a?(Range)
+        last = if index.end.nil?
+                 [TensorStream.shape(self)[0]]
+               else
+                 [index.max + 1]
+               end
+        _op(:strided_slice, self, [index.min], last, [1])
+      else
+        _op(:index, self, index)
+      end
     end
 
     def *(other)

data/lib/tensor_stream/math/math_ops.rb

@@ -0,0 +1,22 @@
+module TensorStream
+  # High level math functions
+  class Maths
+    extend TensorStream::OpHelper
+
+    module MathFunctions
+
+      ##
+      # Normalizes along dimension axis using an L2 norm.
+      def l2_normalize(x, axis: nil, epsilon: 1e-12, name: nil)
+        TensorStream.name_scope(name, "l2_normalize", values: [x]) do |name|
+          x = TensorStream.convert_to_tensor(x, name: "x")
+          square_sum = TensorStream.reduce_sum(TensorStream.square(x), axis, keepdims: true)
+          x_inv_norm = TensorStream.rsqrt(TensorStream.maximum(square_sum, epsilon))
+          TensorStream.multiply(x, x_inv_norm, name: name)
+        end
+      end
+    end
+
+    extend MathFunctions
+  end
+end

data/lib/tensor_stream/math_gradients.rb

@@ -136,7 +136,7 @@ module TensorStream
         when :sparse_softmax_cross_entropy_with_logits
           output = node
           [_broadcast_mul(grad, output[1]), nil]
-         when :zeros_like
+        when :zeros_like
           # non differentiable
           nil
         when :transpose
@@ -165,12 +165,26 @@ module TensorStream
           ts.stack(grad, axis: node.options[:axis])
         when :conv2d
           _Conv2DGrad(node, grad)
+        when :flow_dynamic_stitch
+          num_values = node.inputs.size / 2
+          indices_grad = [nil] * num_values
+
+          inputs = (0...num_values).map { |i| _int32(node, node.inputs[i]) }
+
+          values_grad = inputs.map { |inp| TensorStream.gather(grad, inp) }
+          indices_grad + values_grad
+        when :gather
+          [_op(:gather_grad, grad, node.inputs[1], TensorStream.shape(node.inputs[0])), nil]
         else
           TensorStream::OpMaker.gradient_op(self, node, grad)
         end
       end
     end
 
+    def self._int32(node, x)
+      (node.inputs[0].data_type == :int32 ? x : TensorStream.cast(x, :int32))
+    end
+
     def self._reshape_to_input(node, grad)
       ts.reshape(grad, ts.shape(node.inputs[0]))
     end