tensor_stream-opencl 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  module TensorStream
2
2
  module Opencl
3
- VERSION = "0.2.2"
3
+ VERSION = "0.2.3"
4
4
  end
5
5
  end
data/samples/iris.rb CHANGED
@@ -78,9 +78,9 @@ predict = tf.argmax(yhat, 1)
78
78
  # Backward propagation
79
79
  cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels: y, logits: yhat))
80
80
 
81
- updates = TensorStream::Train::GradientDescentOptimizer.new(0.01).minimize(cost)
81
+ # updates = TensorStream::Train::GradientDescentOptimizer.new(0.01).minimize(cost)
82
82
  # updates = TensorStream::Train::MomentumOptimizer.new(0.01, 0.5, use_nesterov: true).minimize(cost)
83
- # updates = TensorStream::Train::RMSPropOptimizer.new(0.01).minimize(cost)
83
+ updates = TensorStream::Train::RMSPropOptimizer.new(0.01).minimize(cost)
84
84
 
85
85
  # Run SGD
86
86
  sess = tf.session
@@ -0,0 +1,84 @@
1
+ # Model based on https://www.kaggle.com/autuanliuyc/logistic-regression-with-tensorflow
2
+
3
+ require "bundler/setup"
4
+ require 'tensor_stream'
5
+ require 'tensor_stream/opencl'
6
+
7
+ tf = TensorStream
8
+
9
+ rows = File.readlines(File.join("samples","iris.data")).map {|l| l.chomp.split(',') }
10
+
11
+ iris = rows[0...100].shuffle!
12
+
13
+ transformed_data = iris.collect do |row|
14
+ row[0, 4].map(&:to_f)
15
+ end
16
+
17
+ columns = (0..3).map do |i|
18
+ transformed_data.map { |row| row[i] }
19
+ end
20
+
21
+ # Normalize data values before feeding into network
22
+ normalize = -> (val, high, low) { (val - low) / (high - low) } # maps input to float between 0 and 1
23
+
24
+ transformed_data.map! do |row|
25
+ row.map.with_index do |val, j|
26
+ max, min = columns[j].max, columns[j].min
27
+ normalize.(val, max, min)
28
+ end
29
+ end
30
+
31
+
32
+ srand(5)
33
+ seed = 5
34
+ tf.set_random_seed(seed)
35
+
36
+ train_x = transformed_data[0..50].map { |x| x[0..3].map(&:to_f) }
37
+ train_y = iris[0..50].map { |x| x[4] == 'Iris-setosa' ? 0.0 : 1.0 }
38
+
39
+ test_x = transformed_data[51..100].map { |x| x[0..3].map(&:to_f) }
40
+ test_y = iris[51..100].map { |x| x[4] == 'Iris-setosa' ? 0.0 : 1.0 }
41
+
42
+
43
+ A = tf.variable(tf.random_normal([4, 1]))
44
+ b = tf.variable(tf.random_normal([1, 1]))
45
+
46
+ init = tf.global_variables_initializer
47
+ sess = tf.session
48
+ sess.run(init)
49
+
50
+ data = tf.placeholder(:float32, shape: [nil, 4])
51
+ target = tf.placeholder(:float32, shape: [nil, 1])
52
+
53
+ mod = data.dot(A) + b
54
+
55
+ loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits: mod, labels: target))
56
+
57
+ learning_rate = 0.003
58
+ batch_size = 30
59
+ iter_num = 1500
60
+
61
+ optimizer = TensorStream::Train::GradientDescentOptimizer.new(learning_rate)
62
+ goal = optimizer.minimize(loss)
63
+ prediction = tf.round(tf.sigmoid(mod))
64
+ # Bool into float32 type
65
+ correct = tf.cast(tf.equal(prediction, target), :float32)
66
+ # Average
67
+ accuracy = tf.reduce_mean(correct)
68
+
69
+ loss_trace = []
70
+ train_acc = []
71
+ test_acc = []
72
+
73
+ (0..iter_num).each do |epoch|
74
+ batch_train_X = train_x
75
+ batch_train_y = [train_y].transpose
76
+ sess.run(goal, feed_dict: { data => batch_train_X, target => batch_train_y })
77
+
78
+ if epoch % 50 == 0
79
+ temp_loss = sess.run(loss, feed_dict: {data => batch_train_X, target => batch_train_y})
80
+ temp_train_acc = sess.run(accuracy, feed_dict: { data => batch_train_X, target => batch_train_y})
81
+ temp_test_acc = sess.run(accuracy, feed_dict: {data => test_x, target => [test_y].transpose})
82
+ puts "epoch #{epoch}, loss #{temp_loss} train acc: #{temp_train_acc}, test acc: #{temp_test_acc}"
83
+ end
84
+ end
@@ -70,10 +70,13 @@ is_correct = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
70
70
  accuracy = tf.reduce_mean(tf.cast(is_correct, :float32))
71
71
 
72
72
  # training step, learning rate = 0.003
73
- learning_rate = 0.003
74
- train_step = TensorStream::Train::AdamOptimizer.new(learning_rate).minimize(cross_entropy)
73
+ # step for variable learning rate
74
+ step = tf.placeholder(:int32)
75
75
 
76
- sess = tf.session
76
+ lr = tf.constant(0.0001) + tf.train.exponential_decay(0.003, step, 2000, 1/ Math::E)
77
+ train_step = TensorStream::Train::AdamOptimizer.new(lr).minimize(cross_entropy)
78
+
79
+ sess = tf.session(profile_enabled: true)
77
80
  init = tf.global_variables_initializer
78
81
  sess.run(init)
79
82
 
@@ -83,11 +86,13 @@ test_data = { x => mnist.test.images, y_ => mnist.test.labels }
83
86
  (0..10000).each do |i|
84
87
  # load batch of images and correct answers
85
88
  batch_x, batch_y = mnist_train.next_batch(100)
86
- train_data = { x => batch_x, y_ => batch_y }
89
+ train_data = { x => batch_x, y_ => batch_y, step => i }
87
90
 
88
91
  # train
89
92
  sess.run(train_step, feed_dict: train_data)
90
93
  if (i % 50 == 0)
94
+ File.write("profile.json", TensorStream::ReportTool.profile_for(sess).to_json)
95
+ # generate profile
91
96
  # success? add code to print it
92
97
  a_train, c_train = sess.run([accuracy, cross_entropy], feed_dict: train_data)
93
98
 
@@ -30,16 +30,16 @@ N = 30
30
30
 
31
31
 
32
32
  w1 = tf.variable(tf.random_normal([784, K]))
33
- b1 = tf.variable(tf.zeros([K]))
33
+ b1 = tf.variable(tf.ones([K])/10)
34
34
 
35
35
  w2 = tf.variable(tf.random_normal([K, L]))
36
- b2 = tf.variable(tf.zeros([L]))
36
+ b2 = tf.variable(tf.ones([L])/10)
37
37
 
38
38
  w3 = tf.variable(tf.random_normal([L, M]))
39
- b3 = tf.variable(tf.zeros([M]))
39
+ b3 = tf.variable(tf.ones([M])/10)
40
40
 
41
41
  w4 = tf.variable(tf.random_normal([M, N]))
42
- b4 = tf.variable(tf.zeros([N]))
42
+ b4 = tf.variable(tf.ones([N])/10)
43
43
 
44
44
  w5 = tf.variable(tf.random_normal([N, 10]))
45
45
  b5 = tf.variable(tf.zeros([10]))
@@ -57,6 +57,10 @@ y = tf.nn.softmax(ylogits)
57
57
 
58
58
  y_ = tf.placeholder(:float32, shape: [nil, 10])
59
59
 
60
+ # training step, learning rate = 0.003
61
+ # step for variable learning rate
62
+ step = tf.placeholder(:int32)
63
+
60
64
  # cross-entropy loss function (= -sum(Y_i * log(Yi)) ), normalised for batches of 100 images
61
65
  # TensorFlow provides the softmax_cross_entropy_with_logits function to avoid numerical stability
62
66
  # problems with log(0) which is NaN
@@ -67,8 +71,8 @@ is_correct = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
67
71
  accuracy = tf.reduce_mean(tf.cast(is_correct, :float32))
68
72
 
69
73
  # training step, learning rate = 0.003
70
- learning_rate = 0.003
71
- train_step = TensorStream::Train::AdamOptimizer.new(learning_rate).minimize(cross_entropy)
74
+ lr = 0.0001.t + tf.train.exponential_decay(0.003, step, 2000, 1/Math::E)
75
+ train_step = TensorStream::Train::AdamOptimizer.new(lr).minimize(cross_entropy)
72
76
 
73
77
  sess = tf.session
74
78
  # Add ops to save and restore all the variables.
@@ -82,10 +86,11 @@ test_data = { x => mnist.test.images, y_ => mnist.test.labels }
82
86
  (0..1000).each do |i|
83
87
  # load batch of images and correct answers
84
88
  batch_x, batch_y = mnist_train.next_batch(100)
85
- train_data = { x => batch_x, y_ => batch_y }
89
+ train_data = { x => batch_x, y_ => batch_y, step => i }
86
90
 
87
91
  # train
88
92
  sess.run(train_step, feed_dict: train_data)
93
+
89
94
  if (i % 50 == 0)
90
95
  # success? add code to print it
91
96
  a_train, c_train = sess.run([accuracy, cross_entropy], feed_dict: train_data)
@@ -0,0 +1,111 @@
1
+ # A ruby port of the example code discussed by Martin Gorner in
2
+ # "TensorFlow and Deep Learning without a PhD, Part 1 (Google Cloud Next '17)""
3
+ #
4
+ # https://www.youtube.com/watch?v=u4alGiomYP4
5
+ #
6
+ # Requirements:
7
+ # mnist-learn gem
8
+ # opencl_ruby_ffi gem
9
+ require "bundler/setup"
10
+ require 'tensor_stream'
11
+ require 'mnist-learn'
12
+ require 'pry-byebug'
13
+
14
+ # Enable OpenCL hardware accelerated computation, not using OpenCL can be very slow
15
+ require 'tensor_stream/opencl'
16
+
17
+ tf = TensorStream
18
+
19
+ # Import MNIST data
20
+ puts "downloading minst data"
21
+ mnist = Mnist.read_data_sets('/tmp/data', one_hot: true)
22
+ puts "downloading finished"
23
+
24
+ x = tf.placeholder(:float32, shape: [nil, 784])
25
+ y_ = tf.placeholder(:float32, shape: [nil, 10])
26
+
27
+ # Probability of keeping a node during dropout = 1.0 at test time (no dropout) and 0.75 at training time
28
+ pkeep = tf.placeholder(tf.float32)
29
+ # step for variable learning rate
30
+ step = tf.placeholder(:int32)
31
+
32
+ K = 200
33
+ L = 100
34
+ M = 60
35
+ N = 30
36
+
37
+
38
+ w1 = tf.variable(tf.random_normal([784, K]))
39
+ b1 = tf.variable(tf.ones([K])/10)
40
+
41
+ w2 = tf.variable(tf.random_normal([K, L]))
42
+ b2 = tf.variable(tf.ones([L])/10)
43
+
44
+ w3 = tf.variable(tf.random_normal([L, M]))
45
+ b3 = tf.variable(tf.ones([M])/10)
46
+
47
+ w4 = tf.variable(tf.random_normal([M, N]))
48
+ b4 = tf.variable(tf.ones([N])/10)
49
+
50
+ w5 = tf.variable(tf.random_normal([N, 10]))
51
+ b5 = tf.variable(tf.zeros([10]))
52
+
53
+ x_ = tf.reshape(x, [-1, 784])
54
+
55
+ y1 = tf.nn.relu(tf.matmul(x_, w1) + b1)
56
+ y1d = tf.nn.dropout(y1, pkeep)
57
+
58
+ y2 = tf.nn.relu(tf.matmul(y1d, w2) + b2)
59
+ y2d = tf.nn.dropout(y2, pkeep)
60
+
61
+ y3 = tf.nn.relu(tf.matmul(y2d, w3) + b3)
62
+ y3d = tf.nn.dropout(y3, pkeep)
63
+
64
+ y4 = tf.nn.relu(tf.matmul(y3d, w4) + b4)
65
+ y4d = tf.nn.dropout(y4, pkeep)
66
+
67
+ ylogits = tf.matmul(y4d, w5) + b5
68
+
69
+ # model
70
+ y = tf.nn.softmax(ylogits)
71
+
72
+ # cross-entropy loss function (= -sum(Y_i * log(Yi)) ), normalised for batches of 100 images
73
+ # TensorFlow provides the softmax_cross_entropy_with_logits function to avoid numerical stability
74
+ # problems with log(0) which is NaN
75
+ cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits: ylogits, labels: y_)
76
+ cross_entropy = tf.reduce_mean(cross_entropy)*100
77
+
78
+ is_correct = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
79
+ accuracy = tf.reduce_mean(tf.cast(is_correct, :float32))
80
+
81
+ # training step, learning rate = 0.003
82
+ lr = 0.0001.t + tf.train.exponential_decay(0.003, step, 2000, 1/Math::E)
83
+ train_step = TensorStream::Train::AdamOptimizer.new(lr).minimize(cross_entropy)
84
+
85
+ sess = tf.session
86
+ # Add ops to save and restore all the variables.
87
+ saver = tf::Train::Saver.new
88
+ init = tf.global_variables_initializer
89
+
90
+ sess.run(init)
91
+ mnist_train = mnist.train
92
+ test_data = { x => mnist.test.images, y_ => mnist.test.labels, pkeep => 1.0 }
93
+
94
+ (0..1000).each do |i|
95
+ # load batch of images and correct answers
96
+ batch_x, batch_y = mnist_train.next_batch(100)
97
+ train_data = { x => batch_x, y_ => batch_y, step => i, pkeep => 0.75 }
98
+
99
+ # train
100
+ sess.run(train_step, feed_dict: train_data)
101
+
102
+ if (i % 50 == 0)
103
+ # success? add code to print it
104
+ a_train, c_train = sess.run([accuracy, cross_entropy], feed_dict: train_data)
105
+
106
+ # success on test data?
107
+ a_test, c_test = sess.run([accuracy, cross_entropy], feed_dict: test_data)
108
+ puts "#{i} train accuracy #{a_train}, error #{c_train} test accuracy #{a_test}, error #{c_test}"
109
+ end
110
+ end
111
+
data/samples/rnn.rb CHANGED
@@ -75,7 +75,7 @@ losses = logits_series.zip(labels_series).collect do |logits, labels|
75
75
  end
76
76
  total_loss = tf.reduce_mean(losses)
77
77
 
78
- train_step = TensorStream::Train::AdagradOptimizer.new(0.1).minimize(total_loss)
78
+ train_step = TensorStream::Train::AdagradOptimizer.new(0.01).minimize(total_loss)
79
79
 
80
80
  puts "#{tf.get_default_graph.nodes.keys.size} nodes created"
81
81
  zeros_state = tf.zeros([batch_size, state_size]).eval
@@ -38,7 +38,8 @@ Gem::Specification.new do |spec|
38
38
  spec.add_development_dependency "pry-byebug"
39
39
  spec.add_development_dependency "awesome_print"
40
40
  spec.add_development_dependency "mnist-learn"
41
- spec.add_dependency "tensor_stream", "~> 0.9.2"
41
+ spec.add_development_dependency "simplecov"
42
+ spec.add_dependency "tensor_stream", "~> 0.9.7"
42
43
  spec.add_dependency "opencl_ruby_ffi"
43
44
  spec.add_dependency "oily_png"
44
45
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tensor_stream-opencl
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Joseph Dayo
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-10-21 00:00:00.000000000 Z
11
+ date: 2018-11-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -94,20 +94,34 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: simplecov
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
97
111
  - !ruby/object:Gem::Dependency
98
112
  name: tensor_stream
99
113
  requirement: !ruby/object:Gem::Requirement
100
114
  requirements:
101
115
  - - "~>"
102
116
  - !ruby/object:Gem::Version
103
- version: 0.9.2
117
+ version: 0.9.7
104
118
  type: :runtime
105
119
  prerelease: false
106
120
  version_requirements: !ruby/object:Gem::Requirement
107
121
  requirements:
108
122
  - - "~>"
109
123
  - !ruby/object:Gem::Version
110
- version: 0.9.2
124
+ version: 0.9.7
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: opencl_ruby_ffi
113
127
  requirement: !ruby/object:Gem::Requirement
@@ -154,6 +168,7 @@ files:
154
168
  - Rakefile
155
169
  - benchmark/benchmark.rb
156
170
  - benchmark_intel.txt
171
+ - benchmark_ryzen.txt
157
172
  - bin/console
158
173
  - bin/setup
159
174
  - lib/tensor_stream/opencl.rb
@@ -178,6 +193,9 @@ files:
178
193
  - lib/tensor_stream/opencl/kernels/ceil.cl
179
194
  - lib/tensor_stream/opencl/kernels/concat.cl
180
195
  - lib/tensor_stream/opencl/kernels/cond.cl.erb
196
+ - lib/tensor_stream/opencl/kernels/conv2d.cl
197
+ - lib/tensor_stream/opencl/kernels/conv2d_backprop_filter.cl
198
+ - lib/tensor_stream/opencl/kernels/conv2d_backprop_input.cl
181
199
  - lib/tensor_stream/opencl/kernels/cos.cl
182
200
  - lib/tensor_stream/opencl/kernels/div.cl.erb
183
201
  - lib/tensor_stream/opencl/kernels/exp.cl
@@ -189,14 +207,17 @@ files:
189
207
  - lib/tensor_stream/opencl/kernels/log1p.cl
190
208
  - lib/tensor_stream/opencl/kernels/log_softmax.cl
191
209
  - lib/tensor_stream/opencl/kernels/max.cl
210
+ - lib/tensor_stream/opencl/kernels/mean.cl
192
211
  - lib/tensor_stream/opencl/kernels/min.cl
193
212
  - lib/tensor_stream/opencl/kernels/mod.cl
194
213
  - lib/tensor_stream/opencl/kernels/mul.cl
195
214
  - lib/tensor_stream/opencl/kernels/negate.cl
196
215
  - lib/tensor_stream/opencl/kernels/pack.cl
197
216
  - lib/tensor_stream/opencl/kernels/pow.cl
217
+ - lib/tensor_stream/opencl/kernels/prod.cl
198
218
  - lib/tensor_stream/opencl/kernels/real_div.cl
199
219
  - lib/tensor_stream/opencl/kernels/reciprocal.cl
220
+ - lib/tensor_stream/opencl/kernels/relu6.cl
200
221
  - lib/tensor_stream/opencl/kernels/round.cl
201
222
  - lib/tensor_stream/opencl/kernels/sigmoid.cl
202
223
  - lib/tensor_stream/opencl/kernels/sigmoid_grad.cl
@@ -212,6 +233,7 @@ files:
212
233
  - lib/tensor_stream/opencl/kernels/square.cl
213
234
  - lib/tensor_stream/opencl/kernels/squared_difference.cl
214
235
  - lib/tensor_stream/opencl/kernels/sub.cl
236
+ - lib/tensor_stream/opencl/kernels/sum.cl
215
237
  - lib/tensor_stream/opencl/kernels/tan.cl
216
238
  - lib/tensor_stream/opencl/kernels/tanh.cl
217
239
  - lib/tensor_stream/opencl/kernels/tanh_grad.cl
@@ -226,8 +248,10 @@ files:
226
248
  - lib/tensor_stream/opencl/version.rb
227
249
  - samples/iris.data
228
250
  - samples/iris.rb
251
+ - samples/logistic_regression.rb
229
252
  - samples/mnist_data_2.1.rb
230
253
  - samples/mnist_data_2.2.rb
254
+ - samples/mnist_data_2.3.rb
231
255
  - samples/multigpu.rb
232
256
  - samples/nearest_neighbor.rb
233
257
  - samples/rnn.rb