tensor_stream 0.9.8 → 0.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ad97bf7cefbf069baabd2c694489a08117110ab8ce5b6cfecd0af6d43ffd6f2
-  data.tar.gz: c102c1e238d3f914cce40164f65f4dde86735543b8a05c979ff2a49f503ddab4
+  metadata.gz: d311558fd0e0503d46170a6e0d206cfa9eb3d3612143a1b42ef760b548155640
+  data.tar.gz: 5539763e003def63ee40417b9801b10c2694f08cf09f1be10ea3d7fd03cc0a4d
 SHA512:
-  metadata.gz: f14b080e5541f1f34fabc7a46b64cc43ada2a3be667a41f036001baeb4209b13e162bd81af15ca195b18bc39372565ea578af87ed916a85e9aca68e0457c6c20
-  data.tar.gz: 81ee1c521615fb36ded4cdbb94eb855350ca64f1332d579ec866dbc85250fb92711057019ba3a7b74bc4a482ff2d5ed6d130db53baa6becd01ebb9ceff69a05b
+  metadata.gz: 1918a785288e1f25fd75798f5743c924d71e4d7bd63012acdb18df41c5d3ee36b15a66a65fff18740885454b8683479d3e733d4111cbbd3e11fd81cec4fc4765
+  data.tar.gz: fe48b102afb40c36d77f92a3a53ffc155db108a705c01d5fd3cfab5e84d88dfd5c1b7a41f19322430e43335d225a526cf63c757030f7e4ac27f1cc2440797f2f

data/README.md

@@ -2,17 +2,16 @@
 
 # TensorStream
 
-A reimplementation of TensorFlow for ruby. This is a ground up implementation with no dependency on TensorFlow. Effort has been made to make the programming style as near to TensorFlow as possible, comes with a pure ruby evaluator by default with support for an opencl evaluator for large models and datasets.
+An opensource machine learning framework for ruby. Designed to run on a wide variety of ruby implementations (JRuby, TruffleRuby, MRI) as well as an option for High Performance computation (OpenCL).
 
-The goal of this gem is to have a high performance machine learning and compute solution for ruby with support for a wide range of hardware and software configuration.
+This is a framework is heavily influenced by tensorflow and aims to be familiar with tensorflow users. This is a ground up implementation with no dependency on TensorFlow. Effort has been made to make the programming style as near to TensorFlow as possible, comes with a pure ruby evaluator by default with support for an opencl evaluator for large models and datasets.
 
-## Features
+## Goals & Features
 
+- Easy to use - Improve model readability
 - Replicates most of the commonly used low-level tensorflow ops (tf.add, tf.constant, tf.placeholder, tf.matmul, tf.sin etc...)
-- Supports auto-differentiation
-- Provision to use your own opcode evaluator (opencl, sciruby and tensorflow backends planned)
-- Goal is to be as close to TensorFlow in behavior but with some freedom to add ruby specific enhancements (with lots of test cases)
-- (08-08-2018) Load pbtext files from tensorflow (Graph.parse_from_string)
+- Supports auto-differentiation using formal derivation
+- Extensible - use your own opcode evaluator (OpenCL and Pure ruby currently supported)
 
 ## Compatibility
 
@@ -55,7 +54,8 @@ Or install it yourself as:
 
 ## Usage
 
-Usage is similar to how you would use TensorFlow except with ruby syntax
+Usage is similar to how you would use TensorFlow except with ruby syntax.
+There are also enhancements to the syntax to make it as consice as possible.
 
 Linear regression sample:
 
@@ -75,18 +75,24 @@ train_Y = [1.7,2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221,
 
 n_samples = train_X.size
 
-X = tf.placeholder("float")
-Y = tf.placeholder("float")
+# X = tf.placeholder("float")
+X = Float.placeholder
+
+# Y = tf.placeholder("float")
+Y = Float.placeholder
 
 # Set model weights
-W = tf.variable(rand, name: "weight")
-b = tf.variable(rand, name: "bias")
+# W = tf.variable(rand, name: "weight")
+W = rand.t.var name: "weight"
+
+# b = tf.variable(rand, name: "bias")
+b = rand.t.var name: "bias"
 
 # Construct a linear model
 pred = X * W + b
 
 # Mean squared error
-cost = ((pred - Y) ** 2).reduce(:+) / ( 2 * n_samples)
+cost = ((pred - Y) ** 2).reduce / ( 2 * n_samples)
 
 # optimizer = TensorStream::Train::MomentumOptimizer.new(learning_rate, momentum, use_nesterov: true).minimize(cost)
 # optimizer = TensorStream::Train::AdamOptimizer.new(learning_rate).minimize(cost)
@@ -134,7 +140,7 @@ Not all ops are available. Available ops are defined in lib/tensor_stream/ops.rb
 
 There are also certain differences with regards to naming conventions, and named parameters:
 
-# Variables
+# Variables and Constants
 
 To make referencing python examples easier it is recommended to use "tf" as the TensorStream namespace
 
@@ -157,8 +163,15 @@ Ruby
 
 ```ruby
 w = ts.variable(0, name: 'weights')
+c = ts.constant(1.0)
+
+# concise way when initializing using a constant
+w = 0.t.var name: 'weights'
+c = 1.0.t
 ```
 
+Calling .t to Integer, Array and Float types converts it into a tensor
+
 # Shapes
 
 Python
@@ -173,6 +186,10 @@ Ruby
 ```ruby
 x = ts.placeholder(:float32, shape: [1024, 1024])
 x = ts.placeholder(:float32, shape: [nil, 1024])
+
+# Another a bit more terse way
+x = Float.placeholder shape: [1024, 1024]
+y = Float.placeholder shape: [nil, 1024]
 ```
 
 For debugging, each operation or tensor supports the to_math method

data/lib/tensor_stream.rb

@@ -13,7 +13,9 @@ require 'tensor_stream/graph'
 require 'tensor_stream/device'
 require 'tensor_stream/session'
 require 'tensor_stream/tensor_shape'
+require 'tensor_stream/helpers/tensor_mixins'
 require 'tensor_stream/tensor'
+require 'tensor_stream/constant'
 require 'tensor_stream/variable'
 require 'tensor_stream/variable_scope'
 require 'tensor_stream/operation'
@@ -25,8 +27,10 @@ require 'tensor_stream/evaluator/evaluator'
 require 'tensor_stream/graph_serializers/packer'
 require 'tensor_stream/graph_serializers/serializer'
 require 'tensor_stream/graph_deserializers/protobuf'
+require 'tensor_stream/graph_deserializers/yaml_loader'
 require 'tensor_stream/graph_serializers/pbtext'
 require 'tensor_stream/graph_serializers/graphml'
+require 'tensor_stream/graph_serializers/yaml'
 require 'tensor_stream/math_gradients'
 require "tensor_stream/debugging/debugging"
 require 'tensor_stream/utils'

data/lib/tensor_stream/constant.rb

@@ -0,0 +1,41 @@
+module TensorStream
+  # Class that defines a TensorStream variable
+  class Constant < Tensor
+    def initialize(data_type, rank, shape, options = {})
+      setup_initial_state(options)
+      @data_type = data_type
+      @rank = rank
+      @breakpoint = false
+      @shape = TensorShape.new(shape, rank)
+      @value = nil
+      @options = options
+      @is_const = true
+      @internal = options[:internal]
+      @name = [@graph.get_name_scope, options[:name] || build_name].compact.reject(&:empty?).join('/')
+      @given_name = @name
+
+      if options[:value]
+        if options[:value].is_a?(Array)
+          # check if single dimenstion array is passed
+          options[:value] = _reshape(options[:value], shape.reverse.dup) if shape.size >= 2 && !options[:value].empty? && !options[:value][0].is_a?(Array)
+
+          @value = options[:value].map { |v| v.is_a?(Tensor) ? Tensor.cast_dtype(v, @data_type) : v }
+        elsif !shape.empty?
+          @value = _reshape(Tensor.cast_dtype(options[:value], @data_type), shape.dup)
+        else
+          @value = Tensor.cast_dtype(options[:value], @data_type)
+        end
+        @shape = TensorShape.new(shape_eval(@value))
+      end
+
+      @op = Graph.get_default_graph.add_op!(:const, value: @value, data_type: @data_type, internal_name: @name, shape: @shape)
+      @name = @op.name
+    end
+
+    protected
+
+    def build_name
+      "Const"
+    end
+  end
+end

data/lib/tensor_stream/control_flow.rb

@@ -5,11 +5,12 @@ module TensorStream
 
     def initialize(flow_type, inputs, ops = nil, options = {})
       setup_initial_state(options)
-
+      @options = options
       @operation = :"flow_#{flow_type}"
       @inputs = inputs
       @name = [@graph.get_name_scope, options[:name] || set_name].compact.join('/')
       @ops = ops
+      @consumers = Set.new
       @shape = TensorShape.new([inputs.size])
       @graph.add_node(self)
     end

data/lib/tensor_stream/dynamic_stitch.rb

@@ -7,8 +7,10 @@ module TensorStream
       setup_initial_state(options)
 
       @operation = :"flow_#{flow_type}"
-      @inputs = inputs
+      @options = options.merge(n: inputs[0].size)
+      @inputs = inputs.flatten(1).map { |i| TensorStream.convert_to_tensor(i) }.map { |i| i ? i.op : nil }
 
+      @consumers = Set.new
       @data_type = Tensor.detect_type(inputs[1])
       @name = [@graph.get_name_scope, options[:name] || set_name].compact.join('/')
       @ops = ops

data/lib/tensor_stream/evaluator/operation_helpers/array_ops_helper.rb

@@ -16,10 +16,10 @@ module TensorStream
       start_index = start.shift
       current_size = size.shift
       dimen_size = if current_size == -1
-        input.size - 1
-      else
-        start_index + current_size - 1
-      end
+                     input.size - 1
+                   else
+                     start_index + current_size - 1
+                   end
 
       input[start_index..dimen_size].collect do |item|
         if item.is_a?(Array)

data/lib/tensor_stream/evaluator/ruby/array_ops.rb

@@ -11,10 +11,14 @@ module TensorStream
           slice_tensor(input, start.dup, size.dup)
         end
 
-        register_op %i[flow_dynamic_stitch dynamic_stitch] do |_context, _tensor, inputs|
-          indexes, data = inputs
+        register_op %i[flow_dynamic_stitch dynamic_stitch] do |context, tensor, inputs|
+          number_of_indexes = tensor.options[:n]
+          indexes = inputs[0...number_of_indexes]
+          data = inputs[number_of_indexes...inputs.size]
+
           merged = []
-          merge_dynamic_stitch(merged, indexes, data)
+
+          merge_dynamic_stitch(merged, indexes, data, context)
           merged
         end
 
@@ -77,8 +81,6 @@ module TensorStream
         end
 
         register_op :unstack do |_context, tensor, inputs|
-          value = inputs[0]
-
           axis = tensor.options[:axis] || 0
           new_shape = shape_eval(inputs[0])
           rank = new_shape.size - 1
@@ -133,11 +135,9 @@ module TensorStream
 
           if !axis.empty?
             axis.each do |x|
-              if shape[x] == 1
-                shape[x] = nil
-              else
-                raise TensorStream::ValueError, "unable to squeeze dimension that does not have a size of 1"
-              end
+              raise TensorStream::ValueError, "unable to squeeze dimension that does not have a size of 1" if shape[x] != 1
+
+              shape[x] = nil
             end
           else
             shape = shape.map { |s| s == 1 ? nil : s }
@@ -319,7 +319,7 @@ module TensorStream
           end
         end
 
-        register_op :pad do |context, tensor, inputs|
+        register_op :pad do |_context, tensor, inputs|
           arr_pad(inputs[0], inputs[1], tensor.data_type)
         end
 
@@ -333,19 +333,9 @@ module TensorStream
           tile.nil? ? [] : tile
         end
 
-        register_op :cond, noop: true do |context, tensor, inputs|
-          pred = global_eval(tensor, tensor.options[:pred], context)
-
-          if all_true?(pred)
-            global_eval(tensor, inputs[0], context)
-          else
-            global_eval(tensor, inputs[1], context)
-          end
-        end
-
         register_op %i[select where] do |context, tensor, inputs|
-          pred = complete_eval(tensor.options[:pred], context)
-          call_3way_vector_op(pred, inputs[0], inputs[1], context, ->(t, u, v) { t ? u : v })
+          pred = inputs[0]
+          call_3way_vector_op(pred, inputs[1], inputs[2], context, ->(t, u, v) { t ? u : v })
         end
 
         register_op :shape do |_context, tensor, inputs|
@@ -358,6 +348,67 @@ module TensorStream
           end
           TensorStream::Evaluator::OutputGroup.new(shapes, shapes.map { tensor.options[:out_type] })
         end
+
+        register_op :transpose do |_context, _tensor, inputs|
+          shape = shape_eval(inputs[0])
+          rank = get_rank(inputs[0])
+          perm = inputs[1] || (0...rank).to_a.reverse
+
+          if rank == 2 && perm.nil? # use native transpose for general case
+            inputs[0].transpose
+          else
+            arr = inputs[0].flatten
+
+            new_shape = perm.map { |p| shape[p] }
+            new_arr = Array.new(shape.reduce(:*)) { 0 }
+            transpose_with_perm(arr, new_arr, shape, new_shape, perm)
+            TensorShape.reshape(new_arr, new_shape)
+          end
+        end
+
+        register_op :case, noop: true do |context, tensor, _inputs|
+          pred = global_eval(tensor, tensor.inputs[0], context)
+          result = nil
+
+          if tensor.options[:exclusive]
+            p_true = pred.each_with_index.collect { |p, index| [p, index] }.select { |a| a[0] }
+            raise TensorStream::ValueError, "more than one predicate returns true pos #{p_true.map { |a| a[1] }.join(',')}" if p_true.size > 1
+          end
+
+          pred.each_with_index do |p, index|
+            next unless p
+
+            result = global_eval(tensor, tensor.inputs[2 + index], context)
+          end
+
+          result = global_eval(tensor, tensor.inputs[1], context) if result.nil?
+
+          result
+        end
+
+        register_op :case_grad do |_context, tensor, inputs|
+          index, pred, func, grad = inputs
+          if index < 0 && !pred.find { |p| !!p }
+            grad
+          elsif index >= 0 && pred[index]
+            grad
+          else
+            func = -> { int_type?(tensor.data_type) ? 0 : 0.0 }
+            shape = shape_eval(func)
+            generate_vector(shape, generator: func)
+          end
+        end
+
+        def merge_dynamic_stitch(merged, indexes, data, context)
+          indexes.each_with_index do |ind, m|
+            if ind.is_a?(Array)
+              merge_dynamic_stitch(merged, ind, data[m], context)
+            else
+              ind = ind.is_a?(Tensor) ? complete_eval(ind, context) : ind
+              merged[ind] = data[m]
+            end
+          end
+        end
       end
     end
   end

data/lib/tensor_stream/evaluator/ruby/math_ops.rb

@@ -2,24 +2,20 @@ module TensorStream
   module MathOps
     def MathOps.included(klass)
       klass.class_eval do
-        register_op :tanh, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.tanh(t) })
+        register_op :tanh, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { Math.tanh(t) })
         end
 
         register_op :tan, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.tan(t) })
+          call_op(inputs[0], context, ->(t, _b) { Math.tan(t) })
         end
 
-        register_op :atan, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.atan(t) })
+        register_op :atan, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { Math.atan(t) })
         end
 
-        register_op :sec, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.sec(t) })
-        end
-
-        register_op :sin, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.sin(t) })
+        register_op :sin, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { Math.sin(t) })
         end
 
         register_op :add, no_eval: true do |context, tensor, inputs|
@@ -76,67 +72,67 @@ module TensorStream
         end
 
         register_op :round, no_eval: true do |context, _tensor, inputs|
-          call_op(:round, inputs[0], context, ->(t, _b) { t.round })
+          call_op(inputs[0], context, ->(t, _b) { t.round })
         end
 
-        register_op :abs, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { t.abs })
+        register_op :abs, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { t.abs })
         end
 
-        register_op :asin, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.asin(t) })
+        register_op :asin, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { Math.asin(t) })
         end
 
-        register_op :acos, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.acos(t) })
+        register_op :acos, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { Math.acos(t) })
         end
 
         register_op :cos, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.cos(t) })
+          call_op(inputs[0], context, ->(t, _b) { Math.cos(t) })
         end
 
-        register_op :log1p, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.log(1 + t) })
+        register_op :log1p, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { Math.log(1 + t) })
         end
 
-        register_op :log, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { t < 0 ? Float::NAN : Math.log(t) })
+        register_op :log, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { t < 0 ? Float::NAN : Math.log(t) })
         end
 
-        register_op :exp, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.exp(t) })
+        register_op :exp, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { Math.exp(t) })
         end
 
-        register_op :sigmoid, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { sigmoid(t) })
+        register_op :sigmoid, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { sigmoid(t) })
         end
 
-        register_op :sqrt, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { Math.sqrt(t) })
+        register_op :sqrt, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { Math.sqrt(t) })
         end
 
-        register_op :floor, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { t.floor })
+        register_op :floor, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { t.floor })
         end
 
-        register_op :ceil, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { t.ceil })
+        register_op :ceil, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { t.ceil })
         end
 
-        register_op :square, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { t * t })
+        register_op :square, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { t * t })
         end
 
-        register_op :reciprocal, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { 1 / t })
+        register_op :reciprocal, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { 1 / t })
         end
 
         register_op %i[neg negate], no_eval: true do |context, tensor, inputs|
           call_vector_op(tensor, :negate, inputs[0], nil, context, ->(t, _u) { -t })
         end
 
-        register_op :tanh_grad, no_eval: true do |context, tensor, inputs|
-          call_op(tensor, inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
+        register_op :tanh_grad, no_eval: true do |context, _tensor, inputs|
+          call_op(inputs[0], context, ->(t, _b) { 1 - Math.tanh(t) * Math.tanh(t) })
         end
 
         register_op(%i[argmax arg_max]) do |_context, tensor, inputs|
@@ -170,16 +166,13 @@ module TensorStream
         end
 
         register_op :cumprod do |context, tensor, inputs|
-          x = inputs[0]
           c = fp_type?(tensor.data_type) ? 1.0 : 1
           reverse_option = tensor.options[:reverse]
           exclusive = tensor.options[:exclusive]
-
           func = lambda do |arr|
             return c if arr.nil?
-            count = arr.size
-
 
+            count = arr.size
             arr = arr.reverse if reverse_option
             arr = [1] + arr if exclusive
 
@@ -207,6 +200,7 @@ module TensorStream
 
         register_op :prod, noop: true do |context, tensor, _inputs|
           c = fp_type?(tensor.data_type) ? 1.0 : 1
+
           func = lambda do |arr|
             return c if arr.nil?
 
@@ -264,6 +258,14 @@ module TensorStream
         register_op %i[min minimum], noop: true do |context, tensor, inputs|
           call_vector_op(tensor, :min, inputs[0], inputs[1], context, ->(t, u) { [t, u].min })
         end
+
+        def reduction(child_context, tensor, func)
+          val = global_eval(tensor, tensor.inputs[0], child_context)
+          axis = global_eval(tensor, tensor.inputs[1], child_context)
+          keep_dims = global_eval(tensor, tensor.options[:keepdims], child_context)
+
+          reduce(val, axis, keep_dims, func)
+        end
       end
     end
   end