tensor_stream 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 511c8a5edbff0801ad62b74204d964824cc63b26302c032dfa42dd42a1c8483e
-  data.tar.gz: 1f2d961517d3fee1251a22bcf82a660a76076dde6d47d879a2d81715af9a35da
+  metadata.gz: 29a84327103d7c26e81d6d8c0dfa2b92f70941a00e2971ed6bfd0901fa3f516c
+  data.tar.gz: dbd793450ec664358e942eb6c92cff380be5ce5be90539bdf696859ae79446a5
 SHA512:
-  metadata.gz: 01fe16abe086897dd38c4c9a56ce96c9f090dba4220f91ac9bddb892d2bd5760b6518ac939d8df52071b0ecda4f6c7c774fea2eba7532be35ee08fb16a8db04f
-  data.tar.gz: 1927fc911f6e07bb3302030c49ac0b4f2e740f7a15576defd3a1e13b1a1a769c9c31cfed46d0b7c98510f3e5661dd435e87928aed47051124bce1e040d1ab1c5
+  metadata.gz: 62b9963921a2e6b5ff3d7ef4692c84839c6067be6cf6df0f43b189f560d0ccdac24f1df79216356bd06614ed48389085e2c6d64f31a10d15bcb4864218cb15f0
+  data.tar.gz: e0b41ffc15b7c53b26930a5df7088df22bb4e25ff6729609610acd1296be86c71b32234382791ca582afe8e35133d666b33145dac5774d5059fd73b86ba28e9d

data/.gitignore

@@ -8,6 +8,7 @@
 /spec/reports/
 /tmp/
 *.gem
+samples/.ipynb_checkpoints/
 
 # rspec failure tracking
 .rspec_status

data/.rubocop.yml

@@ -1,4 +1,5 @@
 AllCops:
+  TargetRubyVersion: 2.2
   Exclude:
     - samples/*
     - bin/*

data/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 # Specify your gem's dependencies in cl-brains.gemspec
 gemspec

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Joseph Emmanuel Dayo
+Copyright (c) 2018-2019 Joseph Emmanuel Dayo
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -2,9 +2,9 @@
 
 # TensorStream
 
-An opensource machine learning framework for ruby. Designed to run on a wide variety of ruby implementations (JRuby, TruffleRuby, MRI) as well as an option for High Performance computation (OpenCL).
+TensorStream is an opensource framework for machine learning for ruby, its goal is to allow machine learning models to be easily built and run them in various hardware like GPUs and CPUs. It is heavily based on TensorFlow with the goal of being able to easily port its higher level libraries and model examples. As such it is also based on data flow graphs wherein you define computations and data flows between those computations in order to achieve the desired output.
 
-This is a framework is heavily influenced by tensorflow and aims to be familiar with tensorflow users. This is a ground up implementation with no dependency on TensorFlow. Effort has been made to make the programming style as near to TensorFlow as possible, comes with a pure ruby evaluator by default with support for an opencl evaluator for large models and datasets.
+TensorStream is designed to support various backends with a Pure Ruby and OpenCL implementation. These implementations are designed to work together, you can perform training on an OpenCL implementation (where you have a GPU) and then run the resulting trained model on a Pure Ruby implementation where you can deploy anywhere that you can run ruby on. TensorStream has been tested to run on most ruby implementations like MRI, JRuby and TruffleRuby.
 
 ## Goals & Features
 
@@ -12,6 +12,7 @@ This is a framework is heavily influenced by tensorflow and aims to be familiar
 - Replicates most of the commonly used low-level tensorflow ops (tf.add, tf.constant, tf.placeholder, tf.matmul, tf.sin etc...)
 - Supports auto-differentiation using formal derivation
 - Extensible - use your own opcode evaluator (OpenCL and Pure ruby currently supported)
+- wide support - Run on most ruby implementations as well as hardware acceleration on OpenCL supported hardware
 
 ## Compatibility
 
@@ -68,32 +69,30 @@ learning_rate = 0.01
 training_epochs = 1000
 display_step = 50
 
-train_X = [3.3,4.4,5.5,6.71,6.93,4.168,9.779,6.182,7.59,2.167,
-7.042,10.791,5.313,7.997,5.654,9.27,3.1]
-train_Y = [1.7,2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221,
-2.827,3.465,1.65,2.904,2.42,2.94,1.3]
+train_x = [3.3, 4.4, 5.5, 6.71, 6.93, 4.168, 9.779, 6.182, 7.59, 2.167,
+           7.042, 10.791, 5.313, 7.997, 5.654, 9.27, 3.1]
 
-n_samples = train_X.size
+train_y = [1.7, 2.76, 2.09, 3.19, 1.694, 1.573, 3.366, 2.596, 2.53, 1.221,
+           2.827, 3.465, 1.65, 2.904, 2.42, 2.94, 1.3]
 
-# X = tf.placeholder("float")
-X = Float.placeholder
+n_samples = train_x.size
 
-# Y = tf.placeholder("float")
-Y = Float.placeholder
+x_value = Float.placeholder
+y_value = Float.placeholder
 
 # Set model weights
-# W = tf.variable(rand, name: "weight")
-W = rand.t.var name: "weight"
+weight = rand.t.var name: "weight"
 
-# b = tf.variable(rand, name: "bias")
-b = rand.t.var name: "bias"
+bias = rand.t.var name: "bias"
 
 # Construct a linear model
-pred = X * W + b
+pred = x_value * weight + bias
 
 # Mean squared error
-cost = ((pred - Y) ** 2).reduce / ( 2 * n_samples)
+cost = ((pred - y_value)**2).reduce / (2 * n_samples)
 
+# Other optimizers --
+#
 # optimizer = TensorStream::Train::MomentumOptimizer.new(learning_rate, momentum, use_nesterov: true).minimize(cost)
 # optimizer = TensorStream::Train::AdamOptimizer.new(learning_rate).minimize(cost)
 # optimizer = TensorStream::Train::AdadeltaOptimizer.new(1.0).minimize(cost)
@@ -102,27 +101,28 @@ cost = ((pred - Y) ** 2).reduce / ( 2 * n_samples)
 optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate).minimize(cost)
 
 # Initialize the variables (i.e. assign their default value)
-init = tf.global_variables_initializer()
+init = tf.global_variables_initializer
 
 tf.session do |sess|
-    start_time = Time.now
-    sess.run(init)
-    (0..training_epochs).each do |epoch|
-      train_X.zip(train_Y).each do |x,y|
-        sess.run(optimizer, feed_dict: {X => x, Y => y})
-      end
-
-      if (epoch+1) % display_step == 0
-        c = sess.run(cost, feed_dict: { X => train_X, Y => train_Y })
-        puts("Epoch:", '%04d' % (epoch+1), "cost=",  c, \
-            "W=", sess.run(W), "b=", sess.run(b))
-      end
+  start_time = Time.now
+  sess.run(init)
+
+  (0..training_epochs).each do |epoch|
+    train_x.zip(train_y).each do |x, y|
+      sess.run(optimizer, feed_dict: { x_value => x, y_value => y })
     end
 
-    puts("Optimization Finished!")
-    training_cost = sess.run(cost, feed_dict: { X => train_X, Y => train_Y})
-    puts("Training cost=", training_cost, "W=", sess.run(W), "b=", sess.run(b), '\n')
-    puts("time elapsed ", Time.now.to_i - start_time.to_i)
+    if (epoch + 1) % display_step == 0
+      c = sess.run(cost, feed_dict: { x_value => train_x, y_value => train_y })
+      puts("Epoch:", '%04d' % (epoch + 1), "cost=", c, \
+           "W=", sess.run(weight), "b=", sess.run(bias))
+    end
+  end
+
+  puts "Optimization Finished!"
+  training_cost = sess.run(cost, feed_dict: { x_value => train_x, y_value => train_y })
+  puts "Training cost=", training_cost, "W=", sess.run(weight), "b=", sess.run(bias), '\n'
+  puts "time elapsed ", Time.now.to_i - start_time.to_i
 end
 ```
 

data/Rakefile

@@ -1,12 +1,12 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
-require 'rdoc/task'
+require "rdoc/task"
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec
 
 RDoc::Task.new do |rdoc|
   rdoc.main = "README.rdoc"
   rdoc.rdoc_files.include("README.rdoc", "lib   /*.rb")
-end
+end

data/USAGE_GUIDE.md

@@ -0,0 +1,235 @@
+Introduction
+============
+
+This document describes the basic usage of TensorStream and serves as a walthrough with regards to its features, limitations as well as various tools for debugging and development.
+
+Since TensorStream is heavily based on TensorFlow, if you have used TensorFlow before then aside from the syntax a lot of things will be quite familiar.
+
+What is TensorStream?
+--------------------
+
+TensorStream is an opensource framework for machine learning for ruby, its goal is to allow machine learning models to be easily built and run them in various hardware like GPUs and CPUs. It is heavily based on TensorFlow with the goal of being able to easily port its higher level libraries and model examples. As such it is also based on data flow graphs wherein you define computations and data flows between those computations in order to achieve the desired output.
+
+TensorStream is designed to support various backends with a Pure Ruby and OpenCL implementation. These implementations are designed to work together, you can perform training on an OpenCL implementation (where you have a GPU) and then run the resulting trained model on a Pure Ruby implementation where you can deploy anywhere that you can run ruby on. TensorStream has been tested to run on most ruby implementations like MRI, JRuby and TruffleRuby.
+
+Introduction to Tensors
+-----------------------
+
+Tensors are just a mathematical term to describe scalar, single and multidimensional arrays in a consistent manner. Though there is a formal mathematical definition for it, for all intents and purposes these are how data like numbers, strings are represented and structured in order to be fed into operations which in turn process them to be changed into another number or have its structure changed.
+
+Tensors have properties that describe their shape and rank as well as their data type.
+
+Below are examples of Constant Tensors. Their values are immutable and cannot be changed.
+
+```ruby
+t1 = 1.0.t                      # scalar a tensor or rank 0
+t2 = [1.0, 2.1].t               # a float tensor of rank 1
+t3 = [[2.0, 2.1], [2.1, 2.2]].t # a float tensor of rank 2
+t4 = [[2, 2], [3, 3]].t         # an integer tensor of rank 2
+
+# alternatively you can use tensorflow style constant definition
+ts = TensorStream
+t1 = ts.constant(1.0)
+```
+
+Notice that you can create a constant tensor by calling the .t method on an Integer, Float and an Array. You can also use the TensorStream.constant method to achieve the same effect.
+
+tensors can be referenced later by giving it a name (They automatically get a name if you don't give it one)
+
+```ruby
+t1 = 1.0.t(name: 'c1')
+t2 = [5.0].t
+t2.name
+=> "Const"
+
+# tensorflow style
+ts = TensorStream
+t1 = ts.constant(1.0, name: 'c1')
+
+# Reference later
+graph = ts.get_default_graph
+tx = graph['c1']
+tx.run
+=> 1.0
+```
+
+Tensor Shapes
+-------------
+
+The shape to use depends on what the data represents and what computation you want to achieve.
+
+The shape of a tensor describes its structure or describes the dimensions of the array. So for example in order to represent a 28x28 2D grayscale image you would need a tensor with shape [28, 28] with each cell representing a single channel. If instead you have an 28x28 RGB image you would then need a tensor with shape [28, 28, 3], now you need 3 values to represent each pixel. Now what if you need to represent 100 RGB images? then it follows that you need a tensor of size [100, 28, 28, 3]!
+
+Computations
+------------
+
+Naturally the whole point of all of this is to be able to perform computations. TensorStream supports all of the basic math operations you would expect, only beefed up to work with tensors:
+
+```ruby
+t1 = 1.0.t
+t2 = 2.0.t
+sum = t1 + t2
+=> Op(add name: add shape: TensorShape([]) data_type: float32)
+```
+
+Note that sum did not actually compute the "sum" ... yet.
+what happened is that you only defined the data flow graph, in order to get the actual result you need to run it in a session
+
+
+```ruby
+t1 = 1.0.t
+t2 = 2.0.t
+sum = t1 + t2
+=> Op(add name: add shape: TensorShape([]) data_type: float32)
+
+sess = TensorStream.session
+sess.run(sum)
+=> 3.0
+sess.run(t1, t2) # pass multiple tensors/ops
+=> [1.0, 2.0]
+
+# this also works as a shortcut and is equivalent to above
+sum.run
+=> 3.0
+```
+
+TensorStream and TensorFlow (in non eager execution mode) works like this since it uses the dataflow graph to be able to perform gradient computation for machine learning operations. It also uses the graph structure to be able to run that computation in an optimal manner on various hardware like the GPU.
+
+Of course operations on multidimensional arrays work as you would expect
+
+```ruby
+t1 = [1.0, 1.5].t
+t2 = [1.2, 1.5].t
+sum = t1 + t2
+sum.run
+=> [2.2, 3.0].t
+```
+
+There are a wealth of other operations available like reduction for example:
+
+```ruby
+t1 = [1.0, 1.5, 2.0].t
+t1.reduce(:+).run
+=> 4.5
+
+# or tensorflow style
+# ts.reduce_sum(t1)
+```
+
+Broadcast Operations
+--------------------
+
+So Tensor sizes don't have to be the same, you can, in some instances use a different but compatible size in order to perform an operation like below
+
+```ruby
+t1 = [[1.0, 1.5], [1.0, 1.5]].t
+sum = t1 + 1.0
+sum.run
+=>  [[2.0, 2.5], [2.0, 2.5]].t
+```
+
+Here we "broadcasted" a scalar float constant to all cells in a tensor. If these were run on a GPU you can imagine that this operation can be run in parallel.
+
+Below is another way, but using a tensor of rank 1 on a rank 2 tensor:
+
+```ruby
+t1 = [[1.0, 1.5], [1.0, 1.5]].t
+sum = t1 + [1.0, 0.5].t
+sum.run
+=>  [[2.0, 2.0], [2.0, 2.0]].t
+```
+
+In this case we saw that a row by row operation was done instead. There are a number of operations that support broadcasting like multipliation, subtraction, divison etc.
+
+Placeholders and Variables
+--------------------------
+
+There are special types of tensors that are frequently used in building a model in order
+to serve as (Placeholders) for values as well as to store data that can be used in
+succeeding sessions (Variables)
+
+Placeholders are like parameters which take on a value during the time that the model is ran.
+
+For example:
+
+```ruby
+param1 = Float.placeholder
+
+sum = 2.0.t + param1
+sess = TensorStream.session
+sess.run(sum, feed_dict: { param1 => 1.0 })
+=> 3.0
+sess.run(sum, feed_dict: { param1 => 2.0 })
+=> 4.0
+```
+
+Note that NOT passing a value for the placeholder will result in an error.
+
+Variables on the other hand provide persistent data that survives between sessions, however they need to be initialized first otherwise an error will occur.
+
+See below for an example:
+
+```ruby
+ts = TensorStream
+var1 = 1.0.t.var(name: 'var')
+
+acc = var1 + 1
+assign = var1.assign(acc)
+
+# Or tensorflow style
+# var1 = ts.variable(1.0, dtype: :float32, name: 'var1')
+
+# initialize the variables to their initial value
+sess = TensorStream.session
+init = ts.global_variables_initializer
+sess.run(init)
+
+# first run
+sess.run(acc, assign)
+=> [2.0, 2.0]
+sess.run(acc, assign)
+=> [3.0, 3.0]
+```
+
+Variables can be trainable or non-trainable. This property is used by training algorithms to determine if these will be updated during training.
+
+```ruby
+v = TensorStream.variable(1.0, name: 'v', trainable: false)
+```
+
+Graphs
+------
+
+Graphs hold the entire model data structure, each operation defined is stored in a graph which is later used during runtime to perform operations as well as during serialization and deserialization.
+
+When there is no graph present when a tensor is defined, one will automatically be created and will serve as the "default" graph.
+
+Access to the graph can be accomplished using the get_default_graph method.
+
+```ruby
+ts = TensorStream
+graph = ts.get_default_graph
+
+# access nodes
+graph.nodes
+=> {"Const"=>Op(const name: Const shape: TensorShape([]) data_type: float32), "Const_1"=>Op(const name: Const_1 shape: TensorShape([]) data_type: float32)}
+
+```
+
+The graph object can also be used to access collections like a list of variables
+
+```ruby
+vars = graph.get_collection(TensorStream::GraphKeys::GLOBAL_VARIABLES)
+=> [Variable(Variable:0 shape: TensorShape([]) data_type: float32)]
+```
+
+Limitations
+-----------
+
+- Current version only supports dense tensors, meaning for multidimensional arrays each row in a dimension must have the same size. Examples below are not support as of now:
+
+```ruby
+[[1, 2], [1], [2, 3]] # second array has a different size than the others
+```
+
+- The ruby evaluator uses the ruby Float and Integer objects during computation as such the width of the data types (float32 vs float64, int32 vs int64) aren't really used. This however matters with the OpenCL evaluator that uses the width for determining the correct C data type to use in the OpenCL kernels.

data/bin/stubgen

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+# Script to auto generate op stub file from the opdef folder
+
+require "bundler/setup"
+require "tensor_stream"
+require "erb"
+
+target = File.join(__dir__, '..', 'lib', 'tensor_stream', 'generated_stub')
+
+FileUtils.mkdir_p(target)
+
+stub_file = File.join(target, 'ops.rb')
+File.delete(stub_file) if File.exist?(stub_file)
+
+f = File.open(stub_file, 'wb')
+
+template = File.read(File.join(target, 'stub_file.erb'))
+f << ERB.new(template, nil, '%').result(binding)
+f.close

data/exe/model_utils

@@ -2,7 +2,7 @@
 
 require "bundler/setup"
 require "tensor_stream"
-require 'tensor_stream/utils/freezer'
+require "tensor_stream/utils/freezer"
 
 if ARGV[0].nil?
   puts "source checkpoint folder not specified"
@@ -21,4 +21,4 @@ end
 sess = TensorStream.session
 freezer = TensorStream::Freezer.new
 freezer.convert(sess, ARGV[0], ARGV[1])
-exit(0)
+exit(0)

data/lib/tensor_stream.rb

@@ -1,51 +1,52 @@
-require 'tensor_stream/version'
-require 'deep_merge'
-require 'matrix'
-require 'concurrent'
-require 'tensor_stream/exceptions'
-require 'tensor_stream/helpers/op_helper'
-require 'tensor_stream/helpers/string_helper'
-require 'tensor_stream/initializer'
-require 'tensor_stream/graph_keys'
-require 'tensor_stream/types'
-require 'tensor_stream/graph_builder'
-require 'tensor_stream/graph'
-require 'tensor_stream/device'
-require 'tensor_stream/session'
-require 'tensor_stream/tensor_shape'
-require 'tensor_stream/helpers/tensor_mixins'
-require 'tensor_stream/tensor'
-require 'tensor_stream/constant'
-require 'tensor_stream/variable'
-require 'tensor_stream/variable_scope'
-require 'tensor_stream/operation'
-require 'tensor_stream/placeholder'
-require 'tensor_stream/control_flow'
-require 'tensor_stream/dynamic_stitch'
-require 'tensor_stream/nn/nn_ops'
-require 'tensor_stream/evaluator/evaluator'
-require 'tensor_stream/graph_serializers/packer'
-require 'tensor_stream/graph_serializers/serializer'
-require 'tensor_stream/graph_deserializers/protobuf'
-require 'tensor_stream/graph_deserializers/yaml_loader'
-require 'tensor_stream/graph_serializers/pbtext'
-require 'tensor_stream/graph_serializers/graphml'
-require 'tensor_stream/graph_serializers/yaml'
-require 'tensor_stream/math_gradients'
+require "tensor_stream/version"
+require "deep_merge"
+require "matrix"
+require "concurrent"
+require "tensor_stream/exceptions"
+require "tensor_stream/helpers/op_helper"
+require "tensor_stream/helpers/string_helper"
+require "tensor_stream/initializer"
+require "tensor_stream/graph_keys"
+require "tensor_stream/types"
+require "tensor_stream/graph_builder"
+require "tensor_stream/graph"
+require "tensor_stream/device"
+require "tensor_stream/session"
+require "tensor_stream/tensor_shape"
+require "tensor_stream/helpers/tensor_mixins"
+require "tensor_stream/tensor"
+require "tensor_stream/constant"
+require "tensor_stream/variable"
+require "tensor_stream/variable_scope"
+require "tensor_stream/operation"
+require "tensor_stream/placeholder"
+require "tensor_stream/control_flow"
+require "tensor_stream/dynamic_stitch"
+require "tensor_stream/nn/nn_ops"
+require "tensor_stream/evaluator/evaluator"
+require "tensor_stream/graph_serializers/packer"
+require "tensor_stream/graph_serializers/serializer"
+require "tensor_stream/graph_deserializers/protobuf"
+require "tensor_stream/graph_deserializers/yaml_loader"
+require "tensor_stream/graph_serializers/pbtext"
+require "tensor_stream/graph_serializers/graphml"
+require "tensor_stream/graph_serializers/yaml"
+require "tensor_stream/math_gradients"
 require "tensor_stream/debugging/debugging"
-require 'tensor_stream/utils'
-require 'tensor_stream/train/utils'
-require 'tensor_stream/images'
+require "tensor_stream/utils"
+require "tensor_stream/train/utils"
+require "tensor_stream/images"
 
-require 'tensor_stream/profile/report_tool'
+require "tensor_stream/profile/report_tool"
 
 # require 'tensor_stream/libraries/layers'
-require 'tensor_stream/monkey_patches/patch'
-require 'tensor_stream/monkey_patches/integer'
-require 'tensor_stream/monkey_patches/float'
-require 'tensor_stream/monkey_patches/array'
-require 'tensor_stream/ops'
-require 'tensor_stream/trainer'
+require "tensor_stream/monkey_patches/patch"
+require "tensor_stream/monkey_patches/integer"
+require "tensor_stream/monkey_patches/float"
+require "tensor_stream/monkey_patches/array"
+require "tensor_stream/ops"
+require "tensor_stream/trainer"
+require "tensor_stream/op_maker"
 
 # module that exposes TensorStream top level functions
 module TensorStream