tensor_stream-opencl 0.2.2 → 0.2.3

data/lib/tensor_stream/opencl/version.rb

@@ -1,5 +1,5 @@
 module TensorStream
   module Opencl
-    VERSION = "0.2.2"
+    VERSION = "0.2.3"
   end
 end

data/samples/iris.rb

@@ -78,9 +78,9 @@ predict = tf.argmax(yhat, 1)
 # Backward propagation
 cost    = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels: y, logits: yhat))
 
-updates =  TensorStream::Train::GradientDescentOptimizer.new(0.01).minimize(cost)
+# updates =  TensorStream::Train::GradientDescentOptimizer.new(0.01).minimize(cost)
 # updates =  TensorStream::Train::MomentumOptimizer.new(0.01, 0.5, use_nesterov: true).minimize(cost)
-# updates =  TensorStream::Train::RMSPropOptimizer.new(0.01).minimize(cost)
+updates =  TensorStream::Train::RMSPropOptimizer.new(0.01).minimize(cost)
 
 # Run SGD
 sess = tf.session

data/samples/logistic_regression.rb

@@ -0,0 +1,84 @@
+# Model based on https://www.kaggle.com/autuanliuyc/logistic-regression-with-tensorflow
+
+require "bundler/setup"
+require 'tensor_stream'
+require 'tensor_stream/opencl'
+
+tf = TensorStream
+
+rows = File.readlines(File.join("samples","iris.data")).map {|l| l.chomp.split(',') }
+
+iris = rows[0...100].shuffle!
+
+transformed_data = iris.collect do |row|
+  row[0, 4].map(&:to_f)
+end
+
+columns = (0..3).map do |i|
+  transformed_data.map { |row| row[i] }
+end
+
+# Normalize data values before feeding into network
+normalize = -> (val, high, low) { (val - low) / (high - low) } # maps input to float between 0 and 1
+
+transformed_data.map! do |row|
+  row.map.with_index do |val, j|
+    max, min = columns[j].max, columns[j].min
+    normalize.(val, max, min)
+  end
+end
+
+
+srand(5)
+seed = 5
+tf.set_random_seed(seed)
+
+train_x = transformed_data[0..50].map  { |x| x[0..3].map(&:to_f) }
+train_y = iris[0..50].map  { |x| x[4] == 'Iris-setosa' ? 0.0 : 1.0 }
+
+test_x = transformed_data[51..100].map { |x| x[0..3].map(&:to_f) }
+test_y = iris[51..100].map { |x| x[4] == 'Iris-setosa' ? 0.0 : 1.0 }
+
+
+A = tf.variable(tf.random_normal([4, 1]))
+b = tf.variable(tf.random_normal([1, 1]))
+
+init = tf.global_variables_initializer
+sess = tf.session
+sess.run(init)
+
+data = tf.placeholder(:float32, shape: [nil, 4])
+target = tf.placeholder(:float32, shape: [nil, 1])
+
+mod = data.dot(A) + b
+
+loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits: mod, labels: target))
+
+learning_rate = 0.003
+batch_size = 30
+iter_num = 1500
+
+optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate)
+goal = optimizer.minimize(loss)
+prediction = tf.round(tf.sigmoid(mod))
+# Bool into float32 type
+correct = tf.cast(tf.equal(prediction, target), :float32)
+# Average
+accuracy = tf.reduce_mean(correct)
+
+loss_trace = []
+train_acc = []
+test_acc = []
+
+(0..iter_num).each do |epoch|
+  batch_train_X = train_x
+  batch_train_y = [train_y].transpose
+  sess.run(goal, feed_dict: { data => batch_train_X, target => batch_train_y })
+
+  if epoch % 50 == 0
+    temp_loss = sess.run(loss, feed_dict: {data => batch_train_X, target => batch_train_y})
+    temp_train_acc = sess.run(accuracy, feed_dict: { data => batch_train_X, target => batch_train_y})
+    temp_test_acc = sess.run(accuracy, feed_dict: {data => test_x, target => [test_y].transpose})
+    puts "epoch #{epoch}, loss #{temp_loss} train acc: #{temp_train_acc}, test acc: #{temp_test_acc}"
+  end
+end

data/samples/mnist_data_2.1.rb

@@ -70,10 +70,13 @@ is_correct = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
 accuracy =  tf.reduce_mean(tf.cast(is_correct, :float32))
 
 # training step, learning rate = 0.003
-learning_rate = 0.003
-train_step = TensorStream::Train::AdamOptimizer.new(learning_rate).minimize(cross_entropy)
+# step for variable learning rate
+step = tf.placeholder(:int32)
 
-sess = tf.session
+lr = tf.constant(0.0001) + tf.train.exponential_decay(0.003, step, 2000, 1/ Math::E)
+train_step = TensorStream::Train::AdamOptimizer.new(lr).minimize(cross_entropy)
+
+sess = tf.session(profile_enabled: true)
 init = tf.global_variables_initializer
 sess.run(init)
 
@@ -83,11 +86,13 @@ test_data = { x => mnist.test.images, y_ => mnist.test.labels }
 (0..10000).each do |i|
   # load batch of images and correct answers
   batch_x, batch_y = mnist_train.next_batch(100)
-  train_data = { x => batch_x, y_ => batch_y }
+  train_data = { x => batch_x, y_ => batch_y, step => i }
 
   # train
   sess.run(train_step, feed_dict: train_data)
   if (i % 50 == 0)
+    File.write("profile.json", TensorStream::ReportTool.profile_for(sess).to_json)
+    # generate profile
     # success? add code to print it
     a_train, c_train = sess.run([accuracy, cross_entropy], feed_dict: train_data)
 

data/samples/mnist_data_2.2.rb

@@ -30,16 +30,16 @@ N = 30
 
 
 w1 = tf.variable(tf.random_normal([784, K]))
-b1 = tf.variable(tf.zeros([K]))
+b1 = tf.variable(tf.ones([K])/10)
 
 w2 = tf.variable(tf.random_normal([K, L]))
-b2 = tf.variable(tf.zeros([L]))
+b2 = tf.variable(tf.ones([L])/10)
 
 w3 = tf.variable(tf.random_normal([L, M]))
-b3 = tf.variable(tf.zeros([M]))
+b3 = tf.variable(tf.ones([M])/10)
 
 w4 = tf.variable(tf.random_normal([M, N]))
-b4 = tf.variable(tf.zeros([N]))
+b4 = tf.variable(tf.ones([N])/10)
 
 w5 = tf.variable(tf.random_normal([N, 10]))
 b5 = tf.variable(tf.zeros([10]))
@@ -57,6 +57,10 @@ y = tf.nn.softmax(ylogits)
 
 y_ = tf.placeholder(:float32, shape: [nil, 10])
 
+# training step, learning rate = 0.003
+# step for variable learning rate
+step = tf.placeholder(:int32)
+
 # cross-entropy loss function (= -sum(Y_i * log(Yi)) ), normalised for batches of 100  images
 # TensorFlow provides the softmax_cross_entropy_with_logits function to avoid numerical stability
 # problems with log(0) which is NaN
@@ -67,8 +71,8 @@ is_correct = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
 accuracy =  tf.reduce_mean(tf.cast(is_correct, :float32))
 
 # training step, learning rate = 0.003
-learning_rate = 0.003
-train_step = TensorStream::Train::AdamOptimizer.new(learning_rate).minimize(cross_entropy)
+lr = 0.0001.t +  tf.train.exponential_decay(0.003, step, 2000, 1/Math::E)
+train_step = TensorStream::Train::AdamOptimizer.new(lr).minimize(cross_entropy)
 
 sess = tf.session
 # Add ops to save and restore all the variables.
@@ -82,10 +86,11 @@ test_data = { x => mnist.test.images, y_ => mnist.test.labels }
 (0..1000).each do |i|
   # load batch of images and correct answers
   batch_x, batch_y = mnist_train.next_batch(100)
-  train_data = { x => batch_x, y_ => batch_y }
+  train_data = { x => batch_x, y_ => batch_y, step => i }
 
   # train
   sess.run(train_step, feed_dict: train_data)
+
   if (i % 50 == 0)
     # success? add code to print it
     a_train, c_train = sess.run([accuracy, cross_entropy], feed_dict: train_data)

data/samples/mnist_data_2.3.rb

@@ -0,0 +1,111 @@
+# A ruby port of the example code discussed by Martin Gorner in
+# "TensorFlow and Deep Learning without a PhD, Part 1 (Google Cloud Next '17)""
+#
+# https://www.youtube.com/watch?v=u4alGiomYP4
+#
+# Requirements:
+#   mnist-learn gem
+#   opencl_ruby_ffi gem
+require "bundler/setup"
+require 'tensor_stream'
+require 'mnist-learn'
+require 'pry-byebug'
+
+# Enable OpenCL hardware accelerated computation, not using OpenCL can be very slow
+require 'tensor_stream/opencl'
+
+tf = TensorStream
+
+# Import MNIST data
+puts "downloading minst data"
+mnist = Mnist.read_data_sets('/tmp/data', one_hot: true)
+puts "downloading finished"
+
+x = tf.placeholder(:float32, shape: [nil, 784])
+y_ = tf.placeholder(:float32, shape: [nil, 10])
+
+# Probability of keeping a node during dropout = 1.0 at test time (no dropout) and 0.75 at training time
+pkeep = tf.placeholder(tf.float32)
+# step for variable learning rate
+step = tf.placeholder(:int32)
+
+K = 200
+L = 100
+M = 60
+N = 30
+
+
+w1 = tf.variable(tf.random_normal([784, K]))
+b1 = tf.variable(tf.ones([K])/10)
+
+w2 = tf.variable(tf.random_normal([K, L]))
+b2 = tf.variable(tf.ones([L])/10)
+
+w3 = tf.variable(tf.random_normal([L, M]))
+b3 = tf.variable(tf.ones([M])/10)
+
+w4 = tf.variable(tf.random_normal([M, N]))
+b4 = tf.variable(tf.ones([N])/10)
+
+w5 = tf.variable(tf.random_normal([N, 10]))
+b5 = tf.variable(tf.zeros([10]))
+
+x_ = tf.reshape(x, [-1, 784])
+
+y1 = tf.nn.relu(tf.matmul(x_, w1) + b1)
+y1d = tf.nn.dropout(y1, pkeep)
+
+y2 = tf.nn.relu(tf.matmul(y1d, w2) + b2)
+y2d = tf.nn.dropout(y2, pkeep)
+
+y3 = tf.nn.relu(tf.matmul(y2d, w3) + b3)
+y3d = tf.nn.dropout(y3, pkeep)
+
+y4 = tf.nn.relu(tf.matmul(y3d, w4) + b4)
+y4d = tf.nn.dropout(y4, pkeep)
+
+ylogits = tf.matmul(y4d, w5) + b5
+
+# model
+y = tf.nn.softmax(ylogits)
+
+# cross-entropy loss function (= -sum(Y_i * log(Yi)) ), normalised for batches of 100  images
+# TensorFlow provides the softmax_cross_entropy_with_logits function to avoid numerical stability
+# problems with log(0) which is NaN
+cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits: ylogits, labels: y_)
+cross_entropy = tf.reduce_mean(cross_entropy)*100
+
+is_correct = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
+accuracy =  tf.reduce_mean(tf.cast(is_correct, :float32))
+
+# training step, learning rate = 0.003
+lr = 0.0001.t +  tf.train.exponential_decay(0.003, step, 2000, 1/Math::E)
+train_step = TensorStream::Train::AdamOptimizer.new(lr).minimize(cross_entropy)
+
+sess = tf.session
+# Add ops to save and restore all the variables.
+saver = tf::Train::Saver.new
+init = tf.global_variables_initializer
+
+sess.run(init)
+mnist_train = mnist.train
+test_data = { x => mnist.test.images, y_ => mnist.test.labels, pkeep => 1.0 }
+
+(0..1000).each do |i|
+  # load batch of images and correct answers
+  batch_x, batch_y = mnist_train.next_batch(100)
+  train_data = { x => batch_x, y_ => batch_y, step => i, pkeep => 0.75 }
+
+  # train
+  sess.run(train_step, feed_dict: train_data)
+
+  if (i % 50 == 0)
+    # success? add code to print it
+    a_train, c_train = sess.run([accuracy, cross_entropy], feed_dict: train_data)
+
+    # success on test data?
+    a_test, c_test = sess.run([accuracy, cross_entropy], feed_dict: test_data)
+    puts "#{i} train accuracy #{a_train}, error #{c_train} test accuracy #{a_test}, error #{c_test}"
+  end
+end
+

data/samples/rnn.rb

@@ -75,7 +75,7 @@ losses = logits_series.zip(labels_series).collect do |logits, labels|
 end
 total_loss = tf.reduce_mean(losses)
 
-train_step = TensorStream::Train::AdagradOptimizer.new(0.1).minimize(total_loss)
+train_step = TensorStream::Train::AdagradOptimizer.new(0.01).minimize(total_loss)
 
 puts "#{tf.get_default_graph.nodes.keys.size} nodes created"
 zeros_state = tf.zeros([batch_size, state_size]).eval

data/tensor_stream-opencl.gemspec

@@ -38,7 +38,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-byebug"
   spec.add_development_dependency "awesome_print"
   spec.add_development_dependency "mnist-learn"
-  spec.add_dependency "tensor_stream", "~> 0.9.2"
+  spec.add_development_dependency "simplecov"
+  spec.add_dependency "tensor_stream", "~> 0.9.7"
   spec.add_dependency "opencl_ruby_ffi"
   spec.add_dependency "oily_png"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tensor_stream-opencl
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Joseph Dayo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-21 00:00:00.000000000 Z
+date: 2018-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,20 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: tensor_stream
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: 0.9.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: 0.9.7
 - !ruby/object:Gem::Dependency
   name: opencl_ruby_ffi
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +168,7 @@ files:
 - Rakefile
 - benchmark/benchmark.rb
 - benchmark_intel.txt
+- benchmark_ryzen.txt
 - bin/console
 - bin/setup
 - lib/tensor_stream/opencl.rb
@@ -178,6 +193,9 @@ files:
 - lib/tensor_stream/opencl/kernels/ceil.cl
 - lib/tensor_stream/opencl/kernels/concat.cl
 - lib/tensor_stream/opencl/kernels/cond.cl.erb
+- lib/tensor_stream/opencl/kernels/conv2d.cl
+- lib/tensor_stream/opencl/kernels/conv2d_backprop_filter.cl
+- lib/tensor_stream/opencl/kernels/conv2d_backprop_input.cl
 - lib/tensor_stream/opencl/kernels/cos.cl
 - lib/tensor_stream/opencl/kernels/div.cl.erb
 - lib/tensor_stream/opencl/kernels/exp.cl
@@ -189,14 +207,17 @@ files:
 - lib/tensor_stream/opencl/kernels/log1p.cl
 - lib/tensor_stream/opencl/kernels/log_softmax.cl
 - lib/tensor_stream/opencl/kernels/max.cl
+- lib/tensor_stream/opencl/kernels/mean.cl
 - lib/tensor_stream/opencl/kernels/min.cl
 - lib/tensor_stream/opencl/kernels/mod.cl
 - lib/tensor_stream/opencl/kernels/mul.cl
 - lib/tensor_stream/opencl/kernels/negate.cl
 - lib/tensor_stream/opencl/kernels/pack.cl
 - lib/tensor_stream/opencl/kernels/pow.cl
+- lib/tensor_stream/opencl/kernels/prod.cl
 - lib/tensor_stream/opencl/kernels/real_div.cl
 - lib/tensor_stream/opencl/kernels/reciprocal.cl
+- lib/tensor_stream/opencl/kernels/relu6.cl
 - lib/tensor_stream/opencl/kernels/round.cl
 - lib/tensor_stream/opencl/kernels/sigmoid.cl
 - lib/tensor_stream/opencl/kernels/sigmoid_grad.cl
@@ -212,6 +233,7 @@ files:
 - lib/tensor_stream/opencl/kernels/square.cl
 - lib/tensor_stream/opencl/kernels/squared_difference.cl
 - lib/tensor_stream/opencl/kernels/sub.cl
+- lib/tensor_stream/opencl/kernels/sum.cl
 - lib/tensor_stream/opencl/kernels/tan.cl
 - lib/tensor_stream/opencl/kernels/tanh.cl
 - lib/tensor_stream/opencl/kernels/tanh_grad.cl
@@ -226,8 +248,10 @@ files:
 - lib/tensor_stream/opencl/version.rb
 - samples/iris.data
 - samples/iris.rb
+- samples/logistic_regression.rb
 - samples/mnist_data_2.1.rb
 - samples/mnist_data_2.2.rb
+- samples/mnist_data_2.3.rb
 - samples/multigpu.rb
 - samples/nearest_neighbor.rb
 - samples/rnn.rb