tensor_stream 0.9.7 → 0.9.8

data/data_actual.json

@@ -0,0 +1,28 @@
+                            
+                            
+                            
+                            
+                            
+    **********              
+    ********************    
+    ********************    
+    ********************    
+    ********************    
+    ********************    
+    ********************    
+         ***************    
+              **********    
+             **********     
+             **********     
+            **********      
+           ***********      
+          ***********       
+          **********        
+         ***********        
+         **********         
+        ***********         
+        **********          
+        **********          
+        *********           
+        *********           
+        *********           

data/data_expected.json

@@ -0,0 +1,28 @@
+                            
+                            
+                            
+                            
+                            
+    **********              
+    ********************    
+    ********************    
+    ********************    
+    ********************    
+    ********************    
+    ********************    
+         ***************    
+              **********    
+             **********     
+             **********     
+            **********      
+           ***********      
+          ***********       
+          **********        
+         ***********        
+         **********         
+        ***********         
+        **********          
+        **********          
+        *********           
+        *********           
+        *********           

data/data_input.json

@@ -0,0 +1,28 @@
+                            
+                            
+                            
+                            
+                            
+                            
+                            
+      ******                
+      ****************      
+      ****************      
+           ***********      
+                  ****      
+                 ****       
+                 ****       
+                ****        
+                ****        
+               ****         
+               ***          
+              ****          
+             ****           
+            *****           
+            ****            
+           *****            
+           ****             
+          *****             
+          *****             
+          ****              
+                            

data/lib/tensor_stream/evaluator/ruby/nn_ops.rb

@@ -214,25 +214,32 @@ module TensorStream
 
         register_op :conv2d do |_context, tensor, inputs|
           filter = inputs[1]
+
           filter_shape = shape_eval(filter)
           strides = tensor.options[:strides]
+          padding_option = tensor.options[:padding]
           height_stride = strides[1]
           width_stride = strides[2]
 
           raise TensorStream::ValueError, " Current implementation does not yet support strides in the batch and depth dimensions." if strides[0] != 1 || strides[3] != 1
 
+          _batch, height, width, _channels = shape_eval(inputs[0])
+          padding = conv2d_padding_options(padding_option, filter_shape, height, width, height_stride, width_stride)
           inputs[0].collect do |image|
-            height, width, _channels = shape_eval(image)
             f_height, f_width, _input_channels, _output_channels = filter_shape
+            (-padding[0]...height).step(height_stride).map do |y|
+              next if (y + f_height) > (height + padding[2])
+
+              (-padding[1]...width).step(width_stride).map do |x|
+                next if (x + f_width) > (width + padding[3])
 
-            (0...height).step(height_stride).map do |y|
-              (0...width).step(width_stride).map do |x|
                 filter_result = (0...f_height).map do |f_y|
                   (0...f_width).map do |f_x|
                     f_element = filter[f_y][f_x]
 
-                    next if x + f_x >= width
-                    next if y + f_y >= height
+                    next if (x + f_x >= width) || (x + f_x < 0)
+                    next if (y + f_y >= height) || (y + f_y < 0)
+
 
                     image[y + f_y][x + f_x].zip(f_element).map do |image_channel, filter_channels|
                       filter_channels.map { |c| image_channel * c }
@@ -241,16 +248,15 @@ module TensorStream
                 end.flatten(2)
 
                 filter_result.transpose.map { |e| e.reduce(:+) }
-              end
-            end
-          end
+              end.compact
+            end.compact
+          end.compact
         end
 
         register_op :conv2d_backprop_input do |_context, tensor, inputs|
           image_shape, filter, grad = inputs
-
           strides = tensor.options[:strides]
-
+          padding_option = tensor.options[:padding]
           height_stride = strides[1]
           width_stride = strides[2]
 
@@ -259,25 +265,34 @@ module TensorStream
           f_height, f_width, _input_channels, output_channels = filter_shape
           batch, height, width, channels = image_shape
 
+          padding = conv2d_padding_options(padding_option, filter_shape, height, width, height_stride, width_stride)
+
           Array.new(batch) do |b|
             image_gradient = TensorShape.reshape(Array.new(height * width * channels) { 0.0 }, [height, width, channels])
 
-            (0...height).step(height_stride).each do |y|
-              (0...width).step(width_stride).each do |x|
-                img_grad = grad[b][y/height_stride][x/width_stride]
+            ((0 - padding[0])...height).step(height_stride).each do |y|
+              next if (y + f_height) > (height + padding[2])
+
+              ((0 - padding[1])...width).step(width_stride).each do |x|
+                next if (x + f_width) > (width + padding[3])
 
                 (0...f_height).each do |f_y|
                   (0...f_width).each do |f_x|
-                    next if x + f_x >= width
-                    next if y + f_y >= height
+                    next if (y + f_y) < 0 || (y + f_y) >= height
+                    next if (x + f_x) < 0 || (x + f_x) >= width
+
+                    img_grad = grad[b][(y + padding[0]) / height_stride][(x + padding[1]) / width_stride]
 
                     channels.times.each do |c|
-                      image_gradient[y + f_y][x + f_x][c] += Array.new(output_channels) do |o_c|
+                      g = Array.new(output_channels) do |o_c|
                         filter[f_y][f_x][c][o_c] * img_grad[o_c]
                       end.reduce(:+)
+
+                      image_gradient[y + f_y][x + f_x][c] += g
                     end
                   end
                 end
+
               end
             end
 
@@ -289,25 +304,28 @@ module TensorStream
           images, filter_shape, grad = inputs
 
           strides = tensor.options[:strides]
+          padding_option = tensor.options[:padding]
           height_stride = strides[1]
           width_stride = strides[2]
 
           filter_gradient_sum = Array.new(filter_shape.reduce(:*)) { 0.0 }
 
+          _batch, height, width, _channels = shape_eval(images)
+          padding = conv2d_padding_options(padding_option, filter_shape, height, width, height_stride, width_stride)
+
           images.each_with_index.map do |image, index|
-            height, width, _channels = shape_eval(image)
             f_height, f_width, input_channels, output_channels = filter_shape
 
-            (0...height).step(height_stride).each do |y|
-              (0...width).step(width_stride).each do |x|
-                image_grad = grad[index][y/height_stride][x/width_stride]
+            ((0 - padding[0])...height).step(height_stride).each do |y|
+              ((0 - padding[1])...width).step(width_stride).each do |x|
                 filter_result = (0...f_height).map do |f_y|
                   (0...f_width).map do |f_x|
-                    next Array.new(input_channels * output_channels) { 0.0 } if x + f_x >= width
-                    next Array.new(input_channels * output_channels) { 0.0 } if y + f_y >= height
+                    next Array.new(input_channels * output_channels) { 0.0 } if x + f_x >= width || (x + f_x < 0) || ((x + f_width) > (width + padding[3]))
+                    next Array.new(input_channels * output_channels) { 0.0 } if y + f_y >= height || (y + f_y < 0) || ((y + f_height) > (height + padding[2]))
 
-                    image[y + f_y][x + f_x].each_with_index.map do |image_channel, c_channel|
-                      output_channels.times.map do |o_c|
+                    image_grad = grad[index][(y + padding[0]) / height_stride][(x + padding[1])/ width_stride]
+                    image[y + f_y][x + f_x].map do |image_channel|
+                      Array.new(output_channels) do |o_c|
                         image_channel * image_grad[o_c]
                       end
                     end
@@ -321,6 +339,32 @@ module TensorStream
 
           TensorShape.reshape(filter_gradient_sum, filter_shape)
         end
+
+
+        def conv2d_padding_options(padding_option, filter_shape, height, width, h_stride, w_stride)
+          case padding_option
+          when 'SAME'
+            [
+              calc_pad(height, h_stride, filter_shape[0]),
+              calc_pad(width, w_stride, filter_shape[1]),
+              calc_pad(height, h_stride, filter_shape[0], true),
+              calc_pad(width, w_stride, filter_shape[1], true)
+            ]
+          when 'VALID'
+            [0, 0, 0, 0]
+          else
+            raise TensorStream::ValueError, "Unsupported padding value #{padding_option}, valid values 'SAME', 'VALID'"
+          end
+        end
+
+        def calc_pad(w, stride, f_shape, ceil = false)
+          r = ((w / stride - 1) * stride - w + f_shape)
+          if ceil
+            r.odd? ? r / 2 + 1 : r / 2
+          else
+            r / 2
+          end
+        end
       end
     end
   end

data/lib/tensor_stream/evaluator/ruby/random_ops.rb

@@ -45,6 +45,58 @@ module TensorStream
           shape = inputs[0] || tensor.shape.shape
           generate_vector(shape, generator: generator)
         end
+
+        
+        register_op :truncated_normal, no_eval: true do |_context, tensor, inputs|
+          seed = tensor.options[:seed]
+          random = _get_randomizer(tensor, seed)
+          r = RandomGaussian.new(tensor.options.fetch(:mean), tensor.options.fetch(:stddev), -> { random.rand })
+          random = _get_randomizer(tensor, seed)
+          generator = -> { r.rand }
+          shape = inputs[0] || tensor.shape.shape
+          random_values = Array.new(shape.reduce(:*) || 1) {
+            generator.call
+          }
+          mean = random_values.reduce(:+) / random_values.size
+
+          # standard deviation
+
+          stddev = Math.sqrt( random_values.map { |v| ( v - mean )**2 }.reduce(:+) / (random_values.size - 1) )
+          minval = random_values.min
+          maxval = random_values.max
+          max_iterations = 100
+
+          if (minval.infinite? && minval < 0.0) || (maxval < mean)
+            # Reverse all calculations. normMin and normMax will be flipped.
+            a = minval
+            minval = maxval
+            maxval = a
+            stddev = -stddev
+          end
+          
+          norm_min = (minval - mean) / stddev;
+          norm_max = (maxval - mean) / stddev;
+          sqrt_factor = Math.sqrt((norm_min * norm_min) + 4.0);
+          cutoff = 2.0 * Math.exp( 0.5 + (norm_min * (norm_min - sqrt_factor)) / 4.0 ) / (norm_min + sqrt_factor)
+          diff = norm_max - norm_min;
+
+          val = random_values.map { |v|
+            iterations = 0
+            pick = v
+            while ( (pick > norm_max) || (pick < norm_min) )
+              pick = generator.call
+              iterations += 1
+              if iterations > 100
+                pick = v
+                break
+              end
+            end
+
+            pick 
+          }
+
+          TensorShape.reshape(val, shape)
+        end
       end
     end
   end

data/lib/tensor_stream/evaluator/ruby_evaluator.rb

@@ -289,7 +289,8 @@ module TensorStream
           # assertions to make sure inferred shapes == actual evaluated shapes
           if tensor.shape.known? && (result.is_a?(Array) || result.is_a?(Float) || result.is_a?(Integer))
             if shape_eval(result) != tensor.shape.shape
-              # raise "assert error #{tensor.name} #{shape_eval(result)} != #{tensor.shape.shape}"
+
+              raise "assert error #{tensor.name} #{shape_eval(result)} != #{tensor.shape.shape}"
             end
           end
 

data/lib/tensor_stream/helpers/infer_shape.rb

@@ -66,14 +66,16 @@ module TensorStream
         return nil if tensor.inputs[0].shape.nil?
 
         input_shape = tensor.inputs[0].shape.shape
-        return new_shape if input_shape.nil?
-        return nil if input_shape.include?(nil)
+        return new_shape if input_shape.nil? && !new_shape.include?(-1) && !new_shape.include?(nil)
+        return nil if input_shape.nil? || input_shape.include?(nil)
+
         TensorShape.fix_inferred_elements(new_shape, input_shape.reduce(:*))
       when :flow_group
         []
-      when :zeros, :ones, :fill, :random_standard_normal, :random_uniform
+      when :zeros, :ones, :fill, :random_standard_normal, :random_uniform, :truncated_normal
         a_shape = tensor.inputs[0] ? tensor.inputs[0].const_value : tensor.options[:shape]
         return nil if a_shape.nil?
+
         a_shape.is_a?(Array) ? a_shape : [a_shape]
       when :zeros_like, :ones_like
         tensor.inputs[0].shape.shape
@@ -177,6 +179,21 @@ module TensorStream
 
         new_shape = tensor.inputs[0].shape.shape.dup
         new_shape[3] = tensor.inputs[1].shape.shape[3]
+
+        # account for stride and padding options
+        strides = tensor.options[:strides]
+
+        case tensor.options[:padding]
+        when 'SAME'
+          new_shape[1] /= strides[1]
+          new_shape[2] /= strides[2]
+        when 'VALID'
+          new_shape[1] = (new_shape[1] - tensor.inputs[1].shape.shape[0]) / strides[1] + 1
+          new_shape[2] = (new_shape[2] - tensor.inputs[1].shape.shape[1]) / strides[2] + 1
+        else
+          raise TensorStream::ValueError, "Invalid padding option only 'SAME', 'VALID' accepted"
+        end
+
         new_shape
       when :conv2d_backprop_input
         return nil unless tensor.inputs[0].value

data/lib/tensor_stream/helpers/op_helper.rb

@@ -71,7 +71,7 @@ module TensorStream
       return false if tensor.shape.nil?
       return false if tensor.shape.shape.nil?
 
-      tensor.shape.shape.each { |s| return false if s.nil? }
+      tensor.shape.shape.each { |s| return false if s.nil? || (s < 0) }
       true
     end
 

data/lib/tensor_stream/operation.rb

@@ -46,7 +46,7 @@ module TensorStream
     def infer_const
       return false if breakpoint
       case operation
-      when :random_standard_normal, :random_uniform, :glorot_uniform, :print, :check_numerics
+      when :random_standard_normal, :random_uniform, :truncated_normal, :glorot_uniform, :print, :check_numerics
         false
       else
         non_const = @inputs.compact.find { |input| !input.is_const }
@@ -62,7 +62,7 @@ module TensorStream
         :boolean
       when :shape, :rank, :shape_n
         options[:out_type] || :int32
-      when :random_standard_normal, :random_uniform, :glorot_uniform
+      when :random_standard_normal, :random_uniform, :glorot_uniform, :truncated_normal
         passed_data_type || :float32
       when :concat
         @inputs[1].data_type

data/lib/tensor_stream/ops.rb

@@ -92,6 +92,13 @@ module TensorStream
       _op(:random_standard_normal, shape, nil, options)
     end
 
+    ##
+    # Outputs random values from a truncated normal distribution.
+    def truncated_normal(shape, dtype: :float32, mean: 0.0, stddev: 1.0, seed: nil, name: nil)
+      options = { dtype: dtype, mean: mean, stddev: stddev, seed: seed, name: name }
+      _op(:truncated_normal, shape, nil, options)
+    end
+
     ##
     # Stops gradient computation.
     #
@@ -114,7 +121,7 @@ module TensorStream
     # This operation returns a 1-D integer tensor representing the shape of input
     def shape(input, name: nil, out_type: :int32)
       return constant(shape_eval(input, out_type), dtype: out_type, name: "Shape/#{name}") if input.is_a?(Array) && !input[0].is_a?(Tensor)
-      return constant(input.shape.shape, dtype: out_type, name: "Shape/#{input.name}") if shape_full_specified(input)
+      return constant(input.shape.shape, dtype: out_type, name: "Shape/#{input.name}_c") if shape_full_specified(input)
 
       _op(:shape, input, name: name, out_type: out_type)
     end

data/lib/tensor_stream/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
-  VERSION = '0.9.7'.freeze
+  VERSION = '0.9.8'.freeze
 
   def self.version
     VERSION

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tensor_stream
 version: !ruby/object:Gem::Version
-  version: 0.9.7
+  version: 0.9.8
 platform: ruby
 authors:
 - Joseph Emmanuel Dayo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-19 00:00:00.000000000 Z
+date: 2018-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -248,6 +248,11 @@ files:
 - benchmark_ryzen_amd.txt
 - bin/console
 - bin/setup
+- data_1.json
+- data_2.json
+- data_actual.json
+- data_expected.json
+- data_input.json
 - lib/tensor_stream.rb
 - lib/tensor_stream/control_flow.rb
 - lib/tensor_stream/debugging/debugging.rb