tensor_stream 0.7.0 → 0.8.0

data/lib/tensor_stream/variable_scope.rb

@@ -8,7 +8,7 @@ module TensorStream
       @initializer = initializer
     end
 
-    def get_variable(name, dtype: nil, shape: nil, initializer: nil, trainable: true, collections: nil)
+    def get_variable(name, dtype: nil, shape: nil, initializer: nil, trainable: true, collections: nil, validate_shape: false)
       TensorStream::Variable.new(dtype || :float32, nil, shape, self, collections: collections, name: name, initializer: initializer, trainable: trainable)
     end
   end

data/lib/tensor_stream/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
-  VERSION = '0.7.0'.freeze
+  VERSION = '0.8.0'.freeze
 
   def self.version
     VERSION

data/samples/iris.rb

@@ -79,7 +79,7 @@ predict = tf.argmax(yhat, 1)
 
 # Backward propagation
 cost    = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels: y, logits: yhat))
-updates =  TensorStream::Train::GradientDescentOptimizer.new(0.01).minimize(cost)
+updates =  TensorStream::Train::MomentumOptimizer.new(0.01, 0.1, use_nesterov: true).minimize(cost)
 
 # Run SGD
 sess = tf.session
@@ -94,6 +94,7 @@ start_time = Time.now
   x_train.size.times do |i|
     sess.run(updates, feed_dict: {X => [x_train[i]], y => [y_train[i]]})
   end
+
   loss = sess.run(cost, feed_dict: { X => x_train, y => y_train })
   puts "epoch: #{epoch}, loss #{loss}"
 end

data/samples/linear_regression.rb

@@ -5,7 +5,8 @@ require 'benchmark'
 tf = TensorStream
 
 learning_rate = 0.01
-training_epochs = 1000
+momentum = 0.5
+training_epochs = 10000
 display_step = 50
 
 train_X = [3.3,4.4,5.5,6.71,6.93,4.168,9.779,6.182,7.59,2.167,
@@ -28,6 +29,7 @@ pred = X * W + b
 # Mean squared error
 cost = ((pred - Y) ** 2).reduce(:+) / ( 2 * n_samples)
 
+# optimizer = TensorStream::Train::MomentumOptimizer.new(learning_rate, momentum).minimize(cost)
 optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate).minimize(cost)
 
 # Initialize the variables (i.e. assign their default value)

data/samples/nearest_neighbor.rb

@@ -5,6 +5,8 @@ This example is using the MNIST database of handwritten digits
 
 Author: Aymeric Damien
 Project: https://github.com/aymericdamien/TensorFlow-Examples/
+
+Make sure to install the mnist-learn gem
 '''
 require "bundler/setup"
 require 'tensor_stream'

data/test_samples/neural_network_raw.py

@@ -0,0 +1,101 @@
+""" Neural Network.
+
+A 2-Hidden Layers Fully Connected Neural Network (a.k.a Multilayer Perceptron)
+implementation with TensorFlow. This example is using the MNIST database
+of handwritten digits (http://yann.lecun.com/exdb/mnist/).
+
+Links:
+    [MNIST Dataset](http://yann.lecun.com/exdb/mnist/).
+
+Author: Aymeric Damien
+Project: https://github.com/aymericdamien/TensorFlow-Examples/
+"""
+
+from __future__ import print_function
+
+# Import MNIST data
+from tensorflow.examples.tutorials.mnist import input_data
+mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
+
+import tensorflow as tf
+
+# Parameters
+learning_rate = 0.1
+num_steps = 500
+batch_size = 128
+display_step = 100
+
+# Network Parameters
+n_hidden_1 = 256 # 1st layer number of neurons
+n_hidden_2 = 256 # 2nd layer number of neurons
+num_input = 784 # MNIST data input (img shape: 28*28)
+num_classes = 10 # MNIST total classes (0-9 digits)
+
+# tf Graph input
+X = tf.placeholder("float", [None, num_input])
+Y = tf.placeholder("float", [None, num_classes])
+
+# Store layers weight & bias
+weights = {
+    'h1': tf.Variable(tf.random_normal([num_input, n_hidden_1])),
+    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
+    'out': tf.Variable(tf.random_normal([n_hidden_2, num_classes]))
+}
+biases = {
+    'b1': tf.Variable(tf.random_normal([n_hidden_1])),
+    'b2': tf.Variable(tf.random_normal([n_hidden_2])),
+    'out': tf.Variable(tf.random_normal([num_classes]))
+}
+
+
+# Create model
+def neural_net(x):
+    # Hidden fully connected layer with 256 neurons
+    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
+    # Hidden fully connected layer with 256 neurons
+    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
+    # Output fully connected layer with a neuron for each class
+    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
+    return out_layer
+
+# Construct model
+logits = neural_net(X)
+prediction = tf.nn.softmax(logits)
+
+# Define loss and optimizer
+loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
+    logits=logits, labels=Y))
+optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
+train_op = optimizer.minimize(loss_op)
+
+# Evaluate model
+correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))
+accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
+
+# Initialize the variables (i.e. assign their default value)
+init = tf.global_variables_initializer()
+
+# Start training
+with tf.Session() as sess:
+
+    # Run the initializer
+    sess.run(init)
+
+    for step in range(1, num_steps+1):
+        batch_x, batch_y = mnist.train.next_batch(batch_size)
+        # Run optimization op (backprop)
+        sess.run(train_op, feed_dict={X: batch_x, Y: batch_y})
+        if step % display_step == 0 or step == 1:
+            # Calculate batch loss and accuracy
+            loss, acc = sess.run([loss_op, accuracy], feed_dict={X: batch_x,
+                                                                 Y: batch_y})
+            print("Step " + str(step) + ", Minibatch Loss= " + \
+                  "{:.4f}".format(loss) + ", Training Accuracy= " + \
+                  "{:.3f}".format(acc))
+
+    print("Optimization Finished!")
+
+    # Calculate accuracy for MNIST test images
+    print("Testing Accuracy:", \
+        sess.run(accuracy, feed_dict={X: mnist.test.images,
+                                      Y: mnist.test.labels}))

data/test_samples/raw_neural_net_sample.rb

@@ -18,13 +18,15 @@ require 'pry-byebug'
 
 tf = TensorStream
 # Import MNIST data
+puts "downloading minst data"
 mnist = Mnist.read_data_sets('/tmp/data', one_hot: true)
+puts "downloading finished"
 
 # Parameters
 learning_rate = 0.1
 num_steps = 500
 batch_size = 128
-display_step = 100
+display_step = 5
 
 # Network Parameters
 n_hidden_1 = 256 # 1st layer number of neurons
@@ -86,9 +88,9 @@ tf.session do |sess|
     # Run the initializer
     sess.run(init)
 
-    print("Testing Accuracy:", \
-        sess.run(accuracy, feed_dict: { X => mnist.test.images,
-                                        Y => mnist.test.labels}))
+    # print("Testing Accuracy:", \
+    #     sess.run(accuracy, feed_dict: { X => mnist.test.images,
+    #                                     Y => mnist.test.labels}))
 
     (1..num_steps+1).each do |step|
         batch_x, batch_y = mnist.train.next_batch(batch_size)

data/test_samples/test2.py

@@ -1,41 +1,87 @@
 import tensorflow as tf
 
-x = tf.constant([[1.0, 0.5, 4.0]])
+batch_x = [
+          [0.686274, 0.10196, 0.6509, 1.0, 0.9686, 0.49803, 0.0, 0.0, 0.0, 0.0],
+          [0.543244, 0.10123, 0.4509, 0.0, 0.6986, 0.39803, 1.0, 0.0, 0.0, 0.0]]
 
-w = tf.constant([[0.4, 0.2],[0.1, 0.45],[0.2, 4.0]])
+batch_y = [
+          [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0],
+          [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0]
+]
 
-w2 = tf.constant([[0.3, 0.2],[0.15, 0.45]])
-w3 = tf.constant([[0.1, 0.1, 1.0, 1.1, 0.4],[0.05, 0.2, 1.0, 1.2, 0.5],])
+num_input = 10
+num_classes = 10
+n_hidden_1 = 4 # 1st layer number of neurons
+n_hidden_2 = 4 # 2nd layer number of neurons
 
-b= tf.constant([4.0, 5.0])
-b2= tf.constant([4.1, 5.1])
-b3 = tf.constant([2.0, 3.1, 1.0, 0.2, 0.2])
+X = batch_x #tf.placeholder(tf.float32, shape=[None, num_input])
+Y = batch_y # tf.placeholder(tf.float32, shape=[None, num_classes])
 
-matmul_layer_1 = tf.matmul(x, w)
-a = tf.sin(matmul_layer_1 + b)
-matmul_layer_2 = tf.matmul(a, w2)
-matmul_layer_2_add = matmul_layer_2 + b2
-a2 = tf.sin(matmul_layer_2_add)
+h1_init = tf.constant([[0.5937, 0.2343, 1.4332, 0.4395],
+          [-1.0227, -0.6915, 1.2367, 0.3452],
+          [-0.5675, 1.0374, 1.0429, 0.8839],
+          [-0.1066, -0.0469, -1.6317, -1.4836],
+          [0.7835, -3.0105, 1.713, -0.4536],
+          [-0.3076, 1.3662, -0.6537, 0.0905],
+          [-0.2459, 0.2243, -2.7048, 0.848],
+          [0.3589, 0.3542, -0.0959, -1.327],
+          [-0.4685, 0.0844, 0.2794, 2.1275],
+          [-1.0733, 0.6189, 0.845, 0.033]])
 
+h2_init = tf.constant([[0.5012, 1.302, -1.6217, 0.669], [0.1494, -0.7837, -0.2978, 1.7745], [1.9727, -0.5312, -0.7391, 0.9187], [-0.6412, -1.4434, -0.8801, 0.9343]])
+h3_init = tf.constant([[0.5012, 1.302, -1.6217, 0.669, 0.1494, -0.7837, -0.2978, 1.7745, 1.9727, -0.5312],
+  [-0.7391, 0.9187, -0.6412, -1.4434, -0.8801, 0.9343, -0.1665, -0.0032, 0.2959, -2.0488],
+  [-0.9135, 1.0376, 0.8537, 0.4376, 1.3255, -0.5921, -1.4081, 1.0614, -0.5283, 1.1832],
+  [0.7285, -0.7844, 0.1793, -0.5275, -0.4426, -1.4976, 0.4433, 2.2317, -2.0479, 0.7791]])
 
 
-g_matmul_layer_1 =  tf.gradients(matmul_layer_1, [x, w])
-g_sin_a = tf.gradients(a, [b])
-g_matmul_layer_2 =  tf.gradients(matmul_layer_2, [b])
-g_matmul_layer_2_add = tf.gradients(matmul_layer_2_add, [b])
+b1_init = tf.constant([0.1494, -0.7837, -0.2978, 1.7745])
+
+b2_init = tf.constant([1.9727, -0.5312, -0.7391, 0.9187])
+out_init = tf.constant([-0.6412, -1.4434, -0.8801, 0.9343, -0.1665, -0.0032, 0.2959, -2.0488, -0.9135, 1.0376])
+
+h1 = tf.Variable(h1_init, dtype=tf.float32, name='h1')
+h2 = tf.Variable(h2_init, dtype=tf.float32, name='h2')
+h3 = tf.Variable(h3_init, dtype=tf.float32, name='out')
+
+b1 = tf.Variable(b1_init, dtype=tf.float32, name='b1')
+b2 = tf.Variable(b2_init, dtype=tf.float32, name='b2')
+out = tf.Variable(out_init, dtype=tf.float32, name='out2')
+
+layer_1 = tf.add(tf.matmul(X, h1), b1)
+# Hidden fully connected layer with 256 neurons
+layer_2 = tf.add(tf.matmul(layer_1, h2), b2)
+# Output fully connected layer with a neuron for each class
 
 sess = tf.Session()
-s2 = sess.run(g_matmul_layer_2_add)
-g_a2 = tf.gradients(a2, [b], name="final")
 
-print("layer_1 %s", sess.run(g_matmul_layer_1))
-print("layer_2 %s", sess.run(g_matmul_layer_2))
-print("matmul_layer_2_add %s", s2)
-print("g_sin_a %s", sess.run(g_sin_a))
-print("-- %s", sess.run(tf.cos(matmul_layer_2_add) * g_matmul_layer_2_add))
-print("%s", sess.run(g_a2))
+logits = tf.matmul(layer_2, h3) + out
+prediction = tf.nn.softmax(logits)
+
+loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=Y))
+
+optimizer = tf.train.GradientDescentOptimizer(0.01)
+train_op = optimizer.minimize(loss_op)
+init = tf.global_variables_initializer()
+
+sess.run(init)
+# print(sess.run(layer_1))
+tf.gradients(loss_op, [logits])
+print("------------")
+
+print("H1: ", sess.run(h1))
+print("------------ Running train 1")
+# sess.run(train_op, feed_dict={ X: batch_x, Y: batch_y })
+sess.run(train_op)
+print("H1:", sess.run(h1))
+print("H2:", sess.run(h2))
+print("H3:", sess.run(h3))
 
-writer = tf.summary.FileWriter("/home/jedld/graphs/", sess.graph)
-sess.run(g_a2)
-writer.close()
+print(sess.run(b1))
+print(sess.run(b2))
+print(sess.run(out))
 
+# sess.run(train_op, feed_dict={ X: batch_x, Y: batch_y })
+print("------------- Running train 2")
+sess.run(train_op)
+print("H1:", sess.run(h1))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tensor_stream
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Joseph Emmanuel Dayo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-08 00:00:00.000000000 Z
+date: 2018-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -249,6 +249,7 @@ files:
 - lib/tensor_stream/evaluator/opencl/kernels/acos.cl
 - lib/tensor_stream/evaluator/opencl/kernels/add.cl
 - lib/tensor_stream/evaluator/opencl/kernels/apply_gradient.cl
+- lib/tensor_stream/evaluator/opencl/kernels/apply_momentum.cl
 - lib/tensor_stream/evaluator/opencl/kernels/argmax.cl
 - lib/tensor_stream/evaluator/opencl/kernels/argmin.cl
 - lib/tensor_stream/evaluator/opencl/kernels/asin.cl
@@ -270,6 +271,7 @@ files:
 - lib/tensor_stream/evaluator/opencl/kernels/mod.cl
 - lib/tensor_stream/evaluator/opencl/kernels/mul.cl
 - lib/tensor_stream/evaluator/opencl/kernels/negate.cl
+- lib/tensor_stream/evaluator/opencl/kernels/pack.cl
 - lib/tensor_stream/evaluator/opencl/kernels/pow.cl
 - lib/tensor_stream/evaluator/opencl/kernels/real_div.cl
 - lib/tensor_stream/evaluator/opencl/kernels/reciprocal.cl
@@ -319,7 +321,10 @@ files:
 - lib/tensor_stream/tensor.rb
 - lib/tensor_stream/tensor_shape.rb
 - lib/tensor_stream/train/gradient_descent_optimizer.rb
+- lib/tensor_stream/train/momentum_optimizer.rb
+- lib/tensor_stream/train/optimizer.rb
 - lib/tensor_stream/train/saver.rb
+- lib/tensor_stream/train/slot_creator.rb
 - lib/tensor_stream/train/utils.rb
 - lib/tensor_stream/trainer.rb
 - lib/tensor_stream/types.rb
@@ -338,6 +343,7 @@ files:
 - tensor_stream.gemspec
 - test_samples/error.graphml
 - test_samples/gradient_sample.graphml
+- test_samples/neural_network_raw.py
 - test_samples/raw_neural_net_sample.rb
 - test_samples/test.py
 - test_samples/test2.py
@@ -362,7 +368,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: A Pure ruby tensorflow implementation