skynet 0.9.1 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. data/History.txt +99 -0
  2. data/Manifest.txt +10 -9
  3. data/README.txt +74 -7
  4. data/app_generators/skynet_install/skynet_install_generator.rb +26 -22
  5. data/app_generators/skynet_install/templates/migration.rb +11 -5
  6. data/app_generators/skynet_install/templates/skynet +25 -12
  7. data/app_generators/skynet_install/templates/skynet_schema.sql +56 -0
  8. data/bin/skynet +26 -2
  9. data/bin/skynet_install +24 -0
  10. data/bin/skynet_tuplespace_server +13 -0
  11. data/config/hoe.rb +1 -0
  12. data/lib/skynet.rb +3 -0
  13. data/lib/skynet/mapreduce_helper.rb +74 -0
  14. data/lib/skynet/message_queue_adapters/mysql.rb +225 -172
  15. data/lib/skynet/message_queue_adapters/tuple_space.rb +31 -16
  16. data/lib/skynet/skynet_active_record_extensions.rb +78 -46
  17. data/lib/skynet/skynet_config.rb +162 -23
  18. data/lib/skynet/skynet_console.rb +23 -10
  19. data/lib/skynet/skynet_console_helper.rb +61 -58
  20. data/lib/skynet/skynet_job.rb +741 -493
  21. data/lib/skynet/skynet_launcher.rb +5 -1
  22. data/lib/skynet/skynet_manager.rb +106 -49
  23. data/lib/skynet/skynet_message.rb +169 -174
  24. data/lib/skynet/skynet_message_queue.rb +29 -16
  25. data/lib/skynet/skynet_partitioners.rb +92 -0
  26. data/lib/skynet/skynet_ruby_extensions.rb +3 -4
  27. data/lib/skynet/skynet_task.rb +61 -19
  28. data/lib/skynet/skynet_tuplespace_server.rb +0 -2
  29. data/lib/skynet/skynet_worker.rb +73 -51
  30. data/lib/skynet/version.rb +1 -1
  31. data/test/test_active_record_extensions.rb +138 -0
  32. data/test/test_helper.rb +6 -0
  33. data/test/{mysql_message_queue_adaptor_test.rb → test_mysql_message_queue_adapter.rb} +94 -30
  34. data/test/test_skynet.rb +11 -11
  35. data/test/test_skynet_install_generator.rb +0 -4
  36. data/test/test_skynet_job.rb +717 -0
  37. data/test/test_skynet_manager.rb +142 -0
  38. data/test/test_skynet_message.rb +229 -0
  39. data/test/test_skynet_task.rb +24 -0
  40. data/test/{tuplespace_message_queue_test.rb → test_tuplespace_message_queue.rb} +25 -30
  41. data/website/index.html +56 -16
  42. data/website/index.txt +55 -25
  43. data/website/template.rhtml +1 -1
  44. metadata +29 -13
  45. data/app_generators/skynet_install/templates/skynet_console +0 -16
  46. data/bin/skynet_console +0 -9
  47. data/sometest.rb +0 -23
  48. data/test/all_models_test.rb +0 -139
  49. data/test/skynet_manager_test.rb +0 -107
  50. data/test/skynet_message_test.rb +0 -42
  51. data/tmtags +0 -1242
@@ -2,7 +2,7 @@ class Skynet #:nodoc:
2
2
  module VERSION #:nodoc:
3
3
  MAJOR = 0
4
4
  MINOR = 9
5
- TINY = 1
5
+ TINY = 2
6
6
 
7
7
  STRING = [MAJOR, MINOR, TINY].join('.')
8
8
  end
@@ -0,0 +1,138 @@
1
+ require File.dirname(__FILE__) + '/test_helper.rb'
2
+
3
+ class AllModelsTest < Test::Unit::TestCase
4
+
5
+ def test_true
6
+ true
7
+ end
8
+
9
+ # def test_map
10
+ # eachmeth = lambda do |profile|
11
+ # profile.claimed = false
12
+ # profile.save
13
+ # end
14
+ # data = [1,10,{:conditions => "profiles.claimed = 1"},"Profile",eachmeth]
15
+ #
16
+ # assert Profile.find(2).claimed, "claimed"
17
+ #
18
+ # ActiveRecord::Mapreduce.map([data])
19
+ #
20
+ # assert !Profile.find(2).claimed, "not claimed"
21
+ # end
22
+ #
23
+ # def test_each_with_proc
24
+ # Skynet.solo do
25
+ # ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 5).each do |profile|
26
+ # profile.suffix = 'z'
27
+ # profile.save
28
+ # end
29
+ # end
30
+ # assert_equal 'z', Profile.find(2).suffix
31
+ # end
32
+ #
33
+ # def test_big_batch
34
+ # Skynet.solo do
35
+ # ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 200, :model_class => "Profile").each do |profile|
36
+ # profile.suffix = 'z'
37
+ # profile.save
38
+ # end
39
+ # end
40
+ # assert_equal 'z', Profile.find(2).suffix
41
+ # end
42
+ #
43
+ # def test_joins
44
+ # p = Profile.find(1)
45
+ # p.details.zodiac_sign = 'hermit'
46
+ # p.details.save
47
+ # Skynet.solo do
48
+ # ActiveRecord::Mapreduce.find(:all, :conditions => "profile_details.zodiac_sign='hermit'", :joins => "JOIN profile_details ON profiles.id = profile_details.profile_id", :batch_size => 2, :model_class => "Profile").each do |profile|
49
+ # profile.suffix = 'z'
50
+ # profile.save
51
+ # end
52
+ # end
53
+ # assert_equal 'z', Profile.find(1).suffix
54
+ # end
55
+ #
56
+ # def test_stragglers
57
+ # profiles = Profile.find(:all)
58
+ # Skynet.solo do
59
+ # ActiveRecord::Mapreduce.find(:all, :batch_size => profiles.size-1, :model_class => "Profile").each do |profile|
60
+ # profile.suffix = 'z'
61
+ # profile.save
62
+ # end
63
+ # end
64
+ # profiles.last.reload
65
+ # assert_equal 'z', profiles.last.suffix
66
+ # end
67
+ #
68
+ # def test_small_limit
69
+ # Skynet.solo do
70
+ # ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 3).each do |profile|
71
+ # profile.suffix = 'z'
72
+ # profile.save
73
+ # end
74
+ # end
75
+ # assert_equal 'z', Profile.find(2).suffix
76
+ # end
77
+ #
78
+ # def test_each_with_proc_exception
79
+ # Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
80
+ # ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
81
+ # raise "BUSTED" if profile.id == 6
82
+ # profile.suffix = 'z'
83
+ # profile.save
84
+ # end
85
+ # end
86
+ # assert_equal 'z', Profile.find(2).suffix
87
+ # assert_equal nil, Profile.find(6).suffix
88
+ # end
89
+ #
90
+ # def test_each_with_class_exception
91
+ # Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
92
+ # ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(ActiveRecord::MapreduceExTest)
93
+ # end
94
+ # assert_equal 'k', Profile.find(2).suffix
95
+ # assert_equal nil, Profile.find(8).suffix
96
+ # end
97
+ #
98
+ # def test_each_with_class
99
+ # Skynet.solo do
100
+ # ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
101
+ # end
102
+ # assert_equal 'k', Profile.find(2).suffix
103
+ # end
104
+ #
105
+ # def test_distributed_each
106
+ # Skynet.solo do
107
+ # Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
108
+ # profile.suffix = 'gg'
109
+ # profile.save
110
+ # end
111
+ # assert_equal 'gg', Profile.find(2).suffix
112
+ # Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
113
+ # assert_equal 'k', Profile.find(2).suffix
114
+ # end
115
+ # end
116
+ #
117
+ # def test_distributed_each_with_symbol
118
+ # Skynet.solo do
119
+ # Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(:mark_modified)
120
+ # end
121
+ # assert ModifiedProfile.find(:first, :conditions => "profile_id = 2")
122
+ # end
123
+ #
124
+ # def self.each(profile)
125
+ # profile.suffix = 'k'
126
+ # profile.save
127
+ # end
128
+
129
+ end
130
+
131
+ class AllModelsExTest
132
+ def self.each(profile)
133
+ raise "BUSTED" if profile.id == 8
134
+ profile.suffix = 'k'
135
+ profile.save
136
+ end
137
+ end
138
+
data/test/test_helper.rb CHANGED
@@ -1,2 +1,8 @@
1
+ ENV["RAILS_ENV"] = "test"
2
+
1
3
  require 'test/unit'
4
+ require 'rubygems'
2
5
  require File.dirname(__FILE__) + '/../lib/skynet'
6
+ require 'pp'
7
+ require 'mocha'
8
+ require 'functor'
@@ -1,12 +1,17 @@
1
- ENV["RAILS_ENV"] = "test"
1
+ require File.dirname(__FILE__) + '/test_helper.rb'
2
2
 
3
- require 'test/unit'
4
- require '../lib/skynet.rb'
5
-
6
-
7
- # Tests for the partitioners.
8
- #
9
3
  class MysqlMessageQueueTest < Test::Unit::TestCase
4
+ ENV["RAILS_ENV"] = "test"
5
+
6
+ ActiveRecord::Base.establish_connection(
7
+ :adapter => "mysql",
8
+ :host => "localhost",
9
+ :username => "root",
10
+ :password => "",
11
+ :database => "skynet_test"
12
+ )
13
+ ActiveRecord::Base.connection.execute("delete from skynet_message_queues")
14
+ ActiveRecord::Base.connection.execute("delete from skynet_worker_queues")
10
15
 
11
16
  def setup
12
17
  Skynet.configure(
@@ -15,24 +20,35 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
15
20
  :SKYNET_LOG_FILE => STDOUT,
16
21
  :SKYNET_LOG_LEVEL => Logger::ERROR,
17
22
  :MESSAGE_QUEUE_ADAPTER => "Skynet::MessageQueueAdapter::Mysql",
18
- :NEXT_TASK_TIMEOUT => 1
23
+ :MYSQL_NEXT_TASK_TIMEOUT => 1,
24
+ :MYSQL_TEMPERATURE_CHANGE_SLEEP => 1
19
25
  )
20
- mq.clear_outstanding_tasks
21
- mq.clear_worker_status
26
+
27
+ ActiveRecord::Base.establish_connection(
28
+ :adapter => "mysql",
29
+ :host => "localhost",
30
+ :username => "root",
31
+ :password => "",
32
+ :database => "skynet_test"
33
+ )
34
+ ActiveRecord::Base.connection.execute("delete from skynet_message_queues")
35
+ ActiveRecord::Base.connection.execute("delete from skynet_worker_queues")
22
36
 
23
- @worker_message = Skynet::Message.new(
24
- :tasktype=> :task,
25
- :drburi=> "localhost",
26
- :job_id => 1,
27
- :task_id => 2,
28
- :payload => "payload",
37
+ @message_options = {
38
+ :tasktype => "task",
39
+ :job_id => 1,
40
+ :task_id => 2,
41
+ :payload => "payload",
29
42
  :payload_type => "task",
30
- :expiry => 20,
31
- :expire_time => 1095108406.9251,
32
- :iteration => 0,
33
- :name => "name",
34
- :version => 1
35
- )
43
+ :expiry => 20,
44
+ :expire_time => 1095108406.9251,
45
+ :iteration => 0,
46
+ :name => "name",
47
+ :version => 1,
48
+ :queue_id => 0,
49
+ :retry => 1
50
+ }
51
+ @worker_message = Skynet::Message.new(@message_options)
36
52
 
37
53
  end
38
54
 
@@ -40,8 +56,12 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
40
56
 
41
57
  def test_write_message
42
58
  mq.write_message(@worker_message,10)
43
- message = SkynetMessageQueue.find(:all).first
44
- assert_equal @worker_message.expiry.to_f, 20
59
+ message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
60
+ assert_equal @worker_message.expiry.to_f, 20
61
+ @message_options.each do |key,val|
62
+ next if key == :payload
63
+ assert_equal message.send(key), val
64
+ end
45
65
  end
46
66
 
47
67
  def test_template_to_conditions
@@ -62,13 +82,16 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
62
82
  end
63
83
 
64
84
  def test_take_next_task
65
- assert mq.write_message(@worker_message,10)
85
+ assert mq.write_message(@worker_message,10)
86
+ message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
87
+ assert_equal 0, message.iteration
66
88
  task = mq.take_next_task(1,1,:task)
67
89
  assert_equal @worker_message.payload, task.payload
68
-
69
- message = SkynetMessageQueue.find(:first)
90
+
91
+ message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
70
92
  assert_equal 1, message.iteration
71
93
  assert @worker_message.expire_time < message.expire_time
94
+
72
95
  excep = nil
73
96
  begin
74
97
  mq.take_next_task(1)
@@ -85,11 +108,36 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
85
108
  task = mq.take_next_task(1)
86
109
  assert_equal 2, task.task_id
87
110
  assert_equal 0, task.iteration
88
- sleep 0.6
89
111
  ntt = Skynet::Message.next_task_template(1)
90
- next_task = mq.take_next_task(1,0.00001)
112
+ next_task = mq.take_next_task(1,1)
91
113
  assert_equal 2, next_task.task_id
92
114
  assert_equal 1, next_task.iteration
115
+ end
116
+
117
+ def test_take_race_condition
118
+ message = @worker_message.clone
119
+ message.expiry=0.4
120
+ assert mq.write_message(message)
121
+ template = Skynet::Message.next_task_template(1, :task, 0)
122
+
123
+ message_row = mq.send(:find_next_message,template,:task)
124
+ assert !message_row.tran_id
125
+
126
+ assert_equal 1, mq.send(:write_fallback_message,message_row, message)
127
+ assert_equal 0, mq.send(:write_fallback_message,message_row, message)
128
+ assert_equal 0, mq.send(:write_fallback_message,message_row, message)
129
+
130
+ template = Skynet::Message.next_task_template(1, :task, 0)
131
+ another_row = mq.send(:find_next_message,template,:task)
132
+ assert !another_row
133
+ sleep 1
134
+ template = Skynet::Message.next_task_template(1, :task, 0)
135
+ message_row2 = mq.send(:find_next_message,template,:task)
136
+ assert message_row2
137
+ assert_equal message_row.tran_id, message_row2.tran_id
138
+
139
+ message_row2.tran_id = 4
140
+ assert_equal 0, mq.send(:write_fallback_message,message_row2, message)
93
141
  end
94
142
 
95
143
  def test_write_and_take_result
@@ -187,8 +235,24 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
187
235
  assert_equal 10, mq.get_worker_version
188
236
  mq.set_worker_version(11)
189
237
  mq.set_worker_version(12)
190
- assert_equal 1, SkynetWorkerQueue.count(:id, :conditions => "tasktype = 'workerversion'")
238
+ assert_equal 1, SkynetMessageQueue.count(:id, :conditions => "tasktype = 'version'")
191
239
  end
240
+
241
+ def test_version_active?
242
+ mq.set_worker_version(98)
243
+ @worker_message.version = mq.get_worker_version
244
+ mq.write_message(@worker_message,10)
245
+ message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
246
+ assert_equal message.version, 98
247
+ mq.set_worker_version(99)
248
+ assert_equal 99, mq.get_worker_version
249
+ assert_equal true, mq.version_active?(98)
250
+ assert_equal true, mq.version_active?(99)
251
+ message.destroy
252
+ assert_equal false, mq.version_active?(98)
253
+ assert_equal true, mq.version_active?(99)
254
+ end
255
+
192
256
 
193
257
  private
194
258
 
data/test/test_skynet.rb CHANGED
@@ -1,11 +1,11 @@
1
- require File.dirname(__FILE__) + '/test_helper.rb'
2
-
3
- class TestSkynet < Test::Unit::TestCase
4
-
5
- def setup
6
- end
7
-
8
- def test_truth
9
- assert true
10
- end
11
- end
1
+ require File.dirname(__FILE__) + '/test_helper.rb'
2
+
3
+ class TestSkynet < Test::Unit::TestCase
4
+
5
+ def setup
6
+ end
7
+
8
+ def test_truth
9
+ assert true
10
+ end
11
+ end
@@ -28,17 +28,13 @@ class TestSkynetInstallGenerator < Test::Unit::TestCase
28
28
  def test_generator_without_options
29
29
  run_generator('skynet_install', [APP_ROOT], sources)
30
30
  assert_directory_exists "script"
31
- assert_directory_exists "db/migrate"
32
31
  assert_generated_file "script/skynet"
33
- assert_generated_file "script/skynet_console"
34
32
  end
35
33
 
36
34
  def test_generator_in_rails
37
35
  run_generator('skynet_install', [APP_ROOT], sources, {"--rails"=>''})
38
36
  assert_directory_exists "script"
39
- assert_directory_exists "db/migrate"
40
37
  assert_generated_file "script/skynet"
41
- assert_generated_file "script/skynet_console"
42
38
  end
43
39
 
44
40
  private
@@ -0,0 +1,717 @@
1
+ require File.dirname(__FILE__) + '/test_helper.rb'
2
+
3
+ class SkynetJobTest < Test::Unit::TestCase
4
+
5
+ def setup
6
+ Skynet.configure(
7
+ :ENABLE => false,
8
+ :SKYNET_LOG_FILE => STDOUT,
9
+ :MESSAGE_QUEUE_ADAPTER => "Skynet::MessageQueueAdapter::TupleSpace",
10
+ :SKYNET_LOG_LEVEL => Logger::ERROR,
11
+ :TS_DRBURIS => ["druby://localhost:47999"],
12
+ :TS_USE_RINGSERVER => false
13
+ )
14
+
15
+ Skynet::MessageQueue.any_instance.stubs(:get_worker_version).returns(1)
16
+ Skynet::MessageQueue.any_instance.stubs(:set_worker_version).returns(1)
17
+
18
+ @ts = Rinda::TupleSpace.new
19
+ Skynet::MessageQueueAdapter::TupleSpace.stubs(:get_tuple_space).returns(@ts)
20
+
21
+ @messages = []
22
+ end
23
+
24
+ def test_new_job
25
+ job = Skynet::Job.new(:map_reduce_class => JobMRTester)
26
+ {
27
+ :map_name => "JobMRTester MAP",
28
+ :map_timeout => 60,
29
+ :mappers => 2,
30
+ :name => "JobMRTester MASTER",
31
+ :reducers => 1,
32
+ :reduce_partition => nil,
33
+ :reduce => "JobMRTester",
34
+ :queue_id => 0,
35
+ :start_after => 0,
36
+ :reduce_timeout => 60,
37
+ :reduce_name => "JobMRTester REDUCE",
38
+ :master_timeout => 60,
39
+ :map => "JobMRTester",
40
+ :version => 1,
41
+ :result_timeout => 1200,
42
+ :master_result_timeout => 1200
43
+ }.each do |key, val|
44
+ assert_equal val, job.send(key), key
45
+ end
46
+ end
47
+
48
+ def test_new_async_job
49
+ job = Skynet::AsyncJob.new(:map_reduce_class => JobMRTester)
50
+ {
51
+ :map_timeout => 60,
52
+ :reduce_partition => nil,
53
+ :master_timeout => 60,
54
+ :reduce_timeout => 60,
55
+ :name => "JobMRTester MASTER",
56
+ :version => 1,
57
+ :result_timeout => 1200,
58
+ :map_name => "JobMRTester MAP",
59
+ :reducers => 1,
60
+ :queue_id => 0,
61
+ :reduce_name => "JobMRTester REDUCE",
62
+ :map => "JobMRTester",
63
+ :async => true,
64
+ :master_result_timeout => 1200,
65
+ :mappers => 2,
66
+ :reduce => "JobMRTester",
67
+ :start_after => 0
68
+ }.each do |key, val|
69
+ assert_equal val, job.send(key), key
70
+ end
71
+ end
72
+
73
+ def test_run_async
74
+ job = Skynet::AsyncJob.new(
75
+ :map_reduce_class => JobMRTester,
76
+ :version => 1,
77
+ :map_data => [[1]],
78
+ :mappers => 1
79
+ )
80
+ job.stubs(:use_local_queue?).returns(true)
81
+ assert_equal job.job_id, job.run
82
+ assert_equal 1, job.local_mq.messages.size
83
+ master_job = Skynet::Job.new(job.local_mq.messages.first.payload.process)
84
+ assert ! master_job.async
85
+ assert_equal true, master_job.local_master
86
+ end
87
+
88
+ def test_run_master
89
+ job = Skynet::Job.new(
90
+ :map_reduce_class => JobMRTester,
91
+ :version => 1,
92
+ :map_data => [[1]],
93
+ :mappers => 1,
94
+ :local_master => false
95
+ )
96
+ mq = functor
97
+ received_messages = []
98
+ mq.write_message = lambda do |message,timeout|
99
+ received_messages << message
100
+ end
101
+ mq.get_worker_version = 1
102
+ job.stubs(:mq).returns(mq)
103
+ result_message = functor(:payload => "result", :payload_type => :result, :task_id => 1)
104
+ mq.expects(:take_result).with(job.job_id,120).times(1).returns(result_message)
105
+ assert_equal ["result"], job.run
106
+ assert_equal 1, received_messages.size
107
+ end
108
+
109
+ def test_run
110
+ job = Skynet::Job.new(
111
+ :map_reduce_class => JobMRTester,
112
+ :version => 1,
113
+ :map_data => [1,2,3],
114
+ :mappers => 2,
115
+ :reducers => 2
116
+ )
117
+ job.stubs(:use_local_queue?).returns(true)
118
+ results = job.run
119
+ assert_equal 2, results.size
120
+ assert_equal [1,2,3], results.flatten.sort
121
+ end
122
+
123
+ def test_master_enqueue
124
+ job = Skynet::AsyncJob.new(
125
+ :map_reduce_class => JobMRTester,
126
+ :version => 1,
127
+ :map_data => [[1]],
128
+ :mappers => 1
129
+ )
130
+
131
+ mq = functor
132
+ job.stubs(:mq).returns(mq)
133
+ received_messages = []
134
+ mq.write_message = lambda do |message,timeout|
135
+ received_messages << message
136
+ end
137
+ mq.get_worker_version = 1
138
+
139
+ job.master_enqueue
140
+ assert_equal :master, received_messages.first.payload_type
141
+ assert_equal :master, received_messages.first.payload.map_or_reduce
142
+ master_job = Skynet::Job.new(received_messages.first.payload.process)
143
+ assert_equal JobMRTester.to_s, master_job.map
144
+ assert ! master_job.async
145
+ assert_equal true, master_job.local_master
146
+ end
147
+
148
+ def test_master_results
149
+ job = Skynet::Job.new(
150
+ :map_reduce_class => JobMRTester,
151
+ :version => 1,
152
+ :map_data => [[1]],
153
+ :mappers => 1
154
+ )
155
+
156
+ mq = functor(:payload => "result")
157
+ result_message = functor(:payload => "result", :payload_type => :result, :task_id => 1)
158
+ mq.expects(:take_result).with(job.job_id,120).times(1).returns(result_message)
159
+ job.stubs(:mq).returns(mq)
160
+ results = job.master_results
161
+ assert_equal "result", results.first
162
+ end
163
+
164
+ def test_map_enqueue
165
+ job = Skynet::AsyncJob.new(
166
+ :map_reduce_class => JobMRTester,
167
+ :version => 1,
168
+ :map_data => [1,2,3],
169
+ :mappers => 2
170
+ )
171
+
172
+ mq = functor
173
+ job.stubs(:mq).returns(mq)
174
+ received_messages = []
175
+ mq.write_message = lambda do |message,timeout|
176
+ received_messages << message
177
+ end
178
+ mq.get_worker_version = 1
179
+
180
+ job.map_enqueue
181
+ assert_equal 2, received_messages.size
182
+ assert_equal :task, received_messages.first.payload_type
183
+ assert_equal :map, received_messages.first.payload.map_or_reduce
184
+ end
185
+
186
+ def test_map_results
187
+ job = Skynet::AsyncJob.new(
188
+ :map_reduce_class => JobMRTester,
189
+ :version => 1,
190
+ :map_data => [[1]],
191
+ :mappers => 1
192
+ )
193
+
194
+ mq = functor(:payload => "result")
195
+ i = 0
196
+ result_message = functor(:payload => "result", :payload_type => :result, :task_id => lambda {i += 1})
197
+ mq.expects(:take_result).with(job.job_id,120).times(2).returns(result_message)
198
+ job.stubs(:mq).returns(mq)
199
+ results = job.map_results(2)
200
+ assert_equal "result", results.first
201
+ end
202
+
203
+ def test_local_map_results
204
+ job = Skynet::AsyncJob.new(
205
+ :map_reduce_class => JobMRTester,
206
+ :version => 1,
207
+ :map_data => [1,2],
208
+ :mappers => 2,
209
+ :single => true
210
+ )
211
+
212
+ job.map_enqueue
213
+ assert_equal 2, job.local_mq.messages.size
214
+ results = job.map_results(2)
215
+ assert_equal [[1],[2]], results.sort
216
+ end
217
+
218
+ def test_partition_data
219
+ job = Skynet::AsyncJob.new(
220
+ :map_reduce_class => JobMRTester,
221
+ :version => 1,
222
+ :map_data => [[1]],
223
+ :reducers => 2
224
+ )
225
+
226
+ partitioned_data = job.partition_data([[1],[2],[3]])
227
+ assert_equal [[1, 3], [2]], partitioned_data
228
+ end
229
+
230
+ def test_partition_data_class
231
+ job = Skynet::AsyncJob.new(
232
+ :map_reduce_class => "JobPartitionTest",
233
+ :version => 1,
234
+ :map_data => [[1]],
235
+ :reducers => 2
236
+ )
237
+
238
+ partitioned_data = job.partition_data([[1],[2],[3]])
239
+ assert_equal [[1], [2], [3]], partitioned_data
240
+ end
241
+
242
+ def test_reduce_enqueue
243
+ job = Skynet::AsyncJob.new(
244
+ :map_reduce_class => JobMRTester,
245
+ :version => 1,
246
+ :map_data => [1,2,3],
247
+ :mappers => 2
248
+ )
249
+
250
+ mq = functor
251
+ job.stubs(:mq).returns(mq)
252
+ received_messages = []
253
+ mq.write_message = lambda do |message,timeout|
254
+ received_messages << message
255
+ end
256
+ mq.get_worker_version = 1
257
+
258
+ job.reduce_enqueue([[1, 3], [2]])
259
+ assert_equal 2, received_messages.size
260
+ assert_equal :task, received_messages.first.payload_type
261
+ assert_equal :reduce, received_messages.first.payload.map_or_reduce
262
+ end
263
+
264
+ def test_reduce_results
265
+ job = Skynet::AsyncJob.new(
266
+ :map_reduce_class => JobMRTester,
267
+ :version => 1,
268
+ :map_data => [[1]],
269
+ :mappers => 1
270
+ )
271
+
272
+ mq = functor(:payload => "result")
273
+ i = 0
274
+ result_message = functor(:payload => "result", :payload_type => :result, :task_id => lambda {i += 1})
275
+ mq.expects(:take_result).with(job.job_id,120).times(2).returns(result_message)
276
+ job.stubs(:mq).returns(mq)
277
+ results = job.reduce_results(2)
278
+ assert_equal "result", results.first
279
+ end
280
+
281
+ def test_master_task
282
+ job = Skynet::AsyncJob.new(:map_reduce_class => JobMRTester,:version=>1, :queue_id => 4)
283
+ mt = job.master_task
284
+ assert mt.is_a?(Skynet::Task)
285
+ assert_equal mt.result_timeout, 60
286
+ master_job = Skynet::Job.new(mt.process)
287
+ assert_equal job.map, JobMRTester.to_s
288
+ assert_equal job.reduce, JobMRTester.to_s
289
+ assert_equal nil, job.reduce_partition
290
+ assert_equal master_job.map, JobMRTester.to_s
291
+ assert_equal master_job.reduce, JobMRTester.to_s
292
+ assert_equal nil, master_job.reduce_partition
293
+ assert_equal 4, master_job.queue_id
294
+ Skynet::Job::FIELDS.each do |field|
295
+ case field
296
+ when :async, :local_master
297
+ nil
298
+ when :job_id, :single
299
+ next
300
+ else
301
+ assert_equal job.send(field), master_job.send(field), "Testing #{field}, jobfield: #{job.send(field)} mjobfield: #{master_job.send(field)}"
302
+ end
303
+ end
304
+ end
305
+
306
+ def test_gather_results_with_errors
307
+ job = Skynet::AsyncJob.new(
308
+ :map_reduce_class => JobMRTester,
309
+ :version => 1,
310
+ :map_data => [1],
311
+ :mappers => 1
312
+ )
313
+ map_tasks = job.map_tasks
314
+ mq = mq
315
+ mq.stubs(:get_worker_version).returns(1)
316
+ job.stubs(:mq).returns(mq)
317
+ messages = job.tasks_to_messages(map_tasks)
318
+ message = messages.first.result_message(["works"])
319
+ message2 = Skynet::Message.new(message.to_h.merge(:payload_type => :error, :payload => "error", :task_id => 33))
320
+
321
+ test_message1 = {
322
+ :version =>1,
323
+ :queue_id =>0,
324
+ :iteration =>0,
325
+ :name =>"JobMRTester MAP",
326
+ :tasktype =>:result,
327
+ :expire_time =>0,
328
+ :payload_type =>:result,
329
+ :payload =>["works"],
330
+ :drburi =>nil,
331
+ :expiry =>60,
332
+ :retry =>3
333
+ }
334
+ assert message.task_id.is_a?(Bignum)
335
+ assert message.job_id.is_a?(Bignum)
336
+ test_message1.each do |k,v|
337
+ assert_equal v, message.send(k), k
338
+ end
339
+
340
+ mq.expects(:take_result).with(job.job_id, 120).returns(message,message2).times(2)
341
+ results = nil
342
+ Skynet.silent do
343
+ results = job.gather_results(2, map_tasks.first.result_timeout, map_tasks.first.name)
344
+ end
345
+ assert_equal [["works"]], results
346
+ end
347
+
348
+ def test_map_tasks
349
+ job = Skynet::AsyncJob.new(
350
+ :map_reduce_class => JobMRTester,
351
+ :version => 1,
352
+ :map_data => [1,2,3],
353
+ :map_retry => 7,
354
+ :mappers => 2
355
+ )
356
+ map_tasks = job.map_tasks
357
+ assert_equal 2, map_tasks.size
358
+ assert_equal 7, map_tasks.first.retry
359
+ assert map_tasks[0].task_id != map_tasks[1].task_id
360
+ end
361
+
362
+ def test_reduce_tasks
363
+ job = Skynet::AsyncJob.new(
364
+ :map_reduce_class => JobMRTester,
365
+ :version => 1,
366
+ :map_data => [1,2,3],
367
+ :reduce_retry => 9,
368
+ :reducers => 2
369
+ )
370
+ reduce_tasks = job.reduce_tasks([1,2,3])
371
+ assert_equal 3, reduce_tasks.size
372
+ assert_equal 9, reduce_tasks.first.retry
373
+ assert reduce_tasks[0].task_id != reduce_tasks[1].task_id
374
+ end
375
+
376
+ def test_enqueue_messages
377
+ passed_messages = []
378
+ mq = functor
379
+ mq.write_message = lambda do |m,timeout|
380
+ passed_messages << m
381
+ end
382
+ job = Skynet::Job.new(:map_data => [1,2,3], :map_reduce_class => JobMRTester)
383
+ job.stubs(:mq).returns(mq)
384
+ message1 = Skynet::Message.new(
385
+ :tasktype => :task,
386
+ :task_id => 9,
387
+ :job_id => 8,
388
+ :payload_type => :task,
389
+ :payload => "blah",
390
+ :retry => 3,
391
+ :iteration => 0,
392
+ :version => 1,
393
+ :queue_id => 4,
394
+ :name => "test"
395
+ )
396
+ message2 = Skynet::Message.new(message1.to_h.merge(:name => "test2", :payload => "test2"))
397
+ job.enqueue_messages([message1,message2])
398
+ assert_equal [message1, message2], passed_messages
399
+ end
400
+
401
+
402
+ def test_local_queue_write_message
403
+ local_queue = Skynet::Job::LocalMessageQueue.new
404
+ assert_equal 1, local_queue.get_worker_version
405
+ local_queue.write_message(Skynet::Message.new({}),2)
406
+ assert local_queue.messages.first.is_a?(Skynet::Message)
407
+ end
408
+
409
+ def test_local_queue_take_result
410
+ job = Skynet::AsyncJob.new(
411
+ :map_reduce_class => JobMRTester,
412
+ :queue_id => 6,
413
+ :version => 1,
414
+ :map_data => [1,2,3],
415
+ :mappers => 2,
416
+ :master_retry => 17,
417
+ :master_result_timeout => 1
418
+ )
419
+ tasks = job.map_tasks
420
+ assert_equal 2, tasks.size
421
+ messages = job.tasks_to_messages(tasks)
422
+
423
+ local_queue = Skynet::Job::LocalMessageQueue.new
424
+ messages.each do |message|
425
+ local_queue.write_message(message)
426
+ end
427
+ assert_equal messages, local_queue.messages
428
+ assert_equal [1,3], local_queue.take_result(job.job_id,2).payload
429
+ assert_equal [2], local_queue.take_result(job.job_id,2).payload
430
+ assert_equal 0, local_queue.messages.size
431
+ assert_equal 0, local_queue.results.size
432
+ end
433
+
434
+ def test_run_tasks_locally_errors
435
+ job = Skynet::AsyncJob.new(
436
+ :map_reduce_class => JobMRTester,
437
+ :queue_id => 6,
438
+ :version => 1,
439
+ :map_data => [1,2,3],
440
+ :mappers => 2,
441
+ :master_retry => 17,
442
+ :master_result_timeout => 1
443
+ )
444
+ tasks = job.map_tasks
445
+ assert_equal 2, tasks.size
446
+
447
+ messages = job.tasks_to_messages(tasks)
448
+
449
+ task1 = messages.first.payload
450
+ task1.extend(Functor)
451
+ tries = 0
452
+ task1.define_method(:tries) {@tries}
453
+ task1.define_method(:run) do
454
+ @tries ||= 0
455
+ @tries += 1
456
+ if @tries == 1
457
+ raise Exception
458
+ else
459
+ return task1.data
460
+ end
461
+ end
462
+ messages.first.expects(:payload).returns(task1).times(2)
463
+
464
+ task2 = messages[1].payload
465
+ task2.extend(Functor)
466
+ tries = 0
467
+ task2.define_method(:tries) {@tries}
468
+ task2.define_method(:run) do
469
+ @tries ||= 0
470
+ @tries += 1
471
+ return task2.data
472
+ end
473
+ messages[1].expects(:payload).returns(task2).times(1)
474
+
475
+ local_queue = Skynet::Job::LocalMessageQueue.new
476
+ messages.each do |message|
477
+ local_queue.write_message(message)
478
+ end
479
+
480
+ assert_equal messages, local_queue.messages
481
+ Skynet.configure(:SKYNET_LOG_LEVEL=>Logger::FATAL) do
482
+ assert_equal [1,3], local_queue.take_result(job.job_id,2).payload
483
+ assert_equal [2], local_queue.take_result(job.job_id,2).payload
484
+ end
485
+ assert_equal 2, task1.tries
486
+ assert_equal 1, task2.tries
487
+ assert_equal 0, local_queue.messages.size
488
+ assert_equal 0, local_queue.results.size
489
+ end
490
+
491
+ def test_map_local
492
+ job = Skynet::AsyncJob.new(
493
+ :map_reduce_class => JobMRTester,
494
+ :version => 1,
495
+ :map_data => [1,2,3,4],
496
+ :mappers => 3
497
+ )
498
+
499
+ assert_equal 3, job.map_tasks.size
500
+ assert !job.map_local?
501
+
502
+ job.keep_map_tasks = true
503
+ assert job.map_local?
504
+
505
+ job.keep_map_tasks = 3
506
+ assert job.map_local?
507
+
508
+ job.keep_map_tasks = 2
509
+ assert !job.map_local?
510
+
511
+ job.single = true
512
+ assert job.map_local?
513
+ end
514
+
515
+ def test_reduce_local
516
+ job = Skynet::AsyncJob.new(
517
+ :reduce_reduce_class => JobMRTester,
518
+ :version => 1,
519
+ :map_data => [1,2,3,4],
520
+ :reducers => 3
521
+ )
522
+
523
+ partitioned_data = [[1,2],[3],[4]]
524
+ assert_equal 3, job.reduce_tasks(partitioned_data).size
525
+ assert !job.reduce_local?(job.reduce_tasks(partitioned_data))
526
+
527
+ job.keep_reduce_tasks = true
528
+ assert job.reduce_local?(job.reduce_tasks(partitioned_data))
529
+
530
+ job.keep_reduce_tasks = 3
531
+ assert job.reduce_local?(job.reduce_tasks(partitioned_data))
532
+
533
+ job.keep_reduce_tasks = 2
534
+ assert !job.reduce_local?(job.reduce_tasks(partitioned_data))
535
+
536
+ job.single = true
537
+ assert job.reduce_local?(job.reduce_tasks(partitioned_data))
538
+ end
539
+
540
+
541
+
542
+ def test_keep_map_tasks
543
+ job = Skynet::Job.new(
544
+ :map_reduce_class => JobMRTester,
545
+ :version => 1,
546
+ :map_data => [1,2],
547
+ :mappers => 2,
548
+ :keep_map_tasks => 2
549
+ )
550
+ assert_equal 2, job.map_enqueue
551
+ assert_equal 2, job.local_mq.messages.size
552
+ assert_equal [[1],[2]], job.map_results(2).sort
553
+ assert_equal 0, job.local_mq.messages.size
554
+ assert_equal 0, job.local_mq.results.size
555
+ end
556
+
557
+
558
+ def test_keep_reduce_tasks
559
+ job = Skynet::Job.new(
560
+ :map_reduce_class => JobMRTester,
561
+ :version => 1,
562
+ :map_data => [1,2],
563
+ :reducers => 1,
564
+ :keep_reduce_tasks => 2
565
+ )
566
+ assert_equal 1, job.reduce_enqueue([[1,2]])
567
+ assert_equal 1, job.local_mq.messages.size
568
+ assert_equal [[1,2]], job.reduce_results(1).sort
569
+ assert_equal 0, job.local_mq.messages.size
570
+ assert_equal 0, job.local_mq.results.size
571
+ end
572
+
573
+ def test_mapreduce_helper_mixin
574
+ job = Skynet::Job.new(
575
+ :map_reduce_class => "JobMapreduceHelperTest",
576
+ :version => 1,
577
+ :map_data => [1,2],
578
+ :reducers => 2,
579
+ :keep_reduce_tasks => true,
580
+ :keep_map_tasks => true
581
+ )
582
+ map_results = nil
583
+ results = nil
584
+ Skynet.solo do
585
+ map_results = job.map_results(job.map_enqueue)
586
+ partitioned_data = job.partition_data(map_results)
587
+ results = job.reduce_results(job.reduce_enqueue(partitioned_data))
588
+ end
589
+ assert_equal [2,3], map_results.flatten.sort
590
+ assert_equal [3,4], results.flatten.sort
591
+ end
592
+
593
+ def test_mapreduce_helper_mixin_again
594
+ [JobMRTest2, JobMRTest3].each do |klass|
595
+ job = Skynet::Job.new(
596
+ :mappers => 2,
597
+ :reducers => 1,
598
+ :map_reduce_class => klass,
599
+ :map_data => [
600
+ OpenStruct.new({:created_by => 2}),
601
+ OpenStruct.new({:created_by => 2}),
602
+ OpenStruct.new({:created_by => 3})]
603
+ )
604
+
605
+ map_results = nil
606
+ results = nil
607
+ Skynet.solo do
608
+ map_results = job.map_results(job.map_enqueue)
609
+ partitioned_data = job.partition_data(map_results)
610
+ results = job.reduce_results(job.reduce_enqueue(partitioned_data))
611
+ end
612
+ assert_equal [1, 1, 1, 2, 2, 3], map_results.flatten.sort
613
+ expected_results = {2=>2, 3=>1}
614
+ assert_equal expected_results, results
615
+ end
616
+ end
617
+
618
+
619
+ private
620
+
621
+ def mq
622
+ Skynet::MessageQueueAdapter::TupleSpace.new
623
+ end
624
+
625
+ end
626
+
627
+
628
+ class JobMRTester
629
+ def self.map(datas)
630
+ ret = []
631
+ datas.each do |data|
632
+ if data == :error
633
+ raise Exception.new("something bad happened")
634
+ else
635
+ ret << data
636
+ end
637
+ end
638
+ return ret
639
+ end
640
+
641
+ def self.reduce(datas)
642
+ datas
643
+ end
644
+ end
645
+
646
+ class JobPartitionTest < JobMRTester
647
+
648
+ def self.reduce_partition(post_map_data, reducers)
649
+ return post_map_data.compact
650
+ end
651
+ end
652
+
653
+ class JobMapreduceHelperTest
654
+ include MapreduceHelper
655
+
656
+ def self.map_each(data)
657
+ return data + 1
658
+ end
659
+
660
+ def self.reduce_each(data)
661
+ return data + 1
662
+ end
663
+ end
664
+
665
+ class JobMRTest2
666
+ include MapreduceHelper
667
+
668
+ # def self.map(profiles)
669
+ # result = Array.new
670
+ # profiles.each do |profile|
671
+ # result << [profile.created_by, 1] if profile.created_by
672
+ # end
673
+ # result
674
+ # end
675
+
676
+ def self.map_each(item)
677
+ return [item.created_by, 1] if item.created_by
678
+ end
679
+
680
+ def self.reduce(pairs)
681
+ totals = Hash.new
682
+ pairs.each do |pair|
683
+ created_by, count = pair[0], pair[1]
684
+ totals[created_by] ||= 0
685
+ totals[created_by] += count
686
+ end
687
+ return totals
688
+ end
689
+ end
690
+
691
+ class JobMRTest3
692
+
693
+ include MapreduceHelper
694
+
695
+ def self.map(profiles)
696
+ result = Array.new
697
+ profiles.each do |profile|
698
+ result << [profile.created_by, 1] if profile.created_by
699
+ end
700
+ result
701
+ end
702
+
703
+ # def self.map_each(item)
704
+ # return [item.created_by, 1] if item.created_by
705
+ # end
706
+
707
+ def self.reduce(pairs)
708
+ totals = Hash.new
709
+ pairs.each do |pair|
710
+ created_by, count = pair[0], pair[1]
711
+ totals[created_by] ||= 0
712
+ totals[created_by] += count
713
+ end
714
+
715
+ return totals
716
+ end
717
+ end