skynet 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +99 -0
- data/Manifest.txt +10 -9
- data/README.txt +74 -7
- data/app_generators/skynet_install/skynet_install_generator.rb +26 -22
- data/app_generators/skynet_install/templates/migration.rb +11 -5
- data/app_generators/skynet_install/templates/skynet +25 -12
- data/app_generators/skynet_install/templates/skynet_schema.sql +56 -0
- data/bin/skynet +26 -2
- data/bin/skynet_install +24 -0
- data/bin/skynet_tuplespace_server +13 -0
- data/config/hoe.rb +1 -0
- data/lib/skynet.rb +3 -0
- data/lib/skynet/mapreduce_helper.rb +74 -0
- data/lib/skynet/message_queue_adapters/mysql.rb +225 -172
- data/lib/skynet/message_queue_adapters/tuple_space.rb +31 -16
- data/lib/skynet/skynet_active_record_extensions.rb +78 -46
- data/lib/skynet/skynet_config.rb +162 -23
- data/lib/skynet/skynet_console.rb +23 -10
- data/lib/skynet/skynet_console_helper.rb +61 -58
- data/lib/skynet/skynet_job.rb +741 -493
- data/lib/skynet/skynet_launcher.rb +5 -1
- data/lib/skynet/skynet_manager.rb +106 -49
- data/lib/skynet/skynet_message.rb +169 -174
- data/lib/skynet/skynet_message_queue.rb +29 -16
- data/lib/skynet/skynet_partitioners.rb +92 -0
- data/lib/skynet/skynet_ruby_extensions.rb +3 -4
- data/lib/skynet/skynet_task.rb +61 -19
- data/lib/skynet/skynet_tuplespace_server.rb +0 -2
- data/lib/skynet/skynet_worker.rb +73 -51
- data/lib/skynet/version.rb +1 -1
- data/test/test_active_record_extensions.rb +138 -0
- data/test/test_helper.rb +6 -0
- data/test/{mysql_message_queue_adaptor_test.rb → test_mysql_message_queue_adapter.rb} +94 -30
- data/test/test_skynet.rb +11 -11
- data/test/test_skynet_install_generator.rb +0 -4
- data/test/test_skynet_job.rb +717 -0
- data/test/test_skynet_manager.rb +142 -0
- data/test/test_skynet_message.rb +229 -0
- data/test/test_skynet_task.rb +24 -0
- data/test/{tuplespace_message_queue_test.rb → test_tuplespace_message_queue.rb} +25 -30
- data/website/index.html +56 -16
- data/website/index.txt +55 -25
- data/website/template.rhtml +1 -1
- metadata +29 -13
- data/app_generators/skynet_install/templates/skynet_console +0 -16
- data/bin/skynet_console +0 -9
- data/sometest.rb +0 -23
- data/test/all_models_test.rb +0 -139
- data/test/skynet_manager_test.rb +0 -107
- data/test/skynet_message_test.rb +0 -42
- data/tmtags +0 -1242
data/lib/skynet/version.rb
CHANGED
@@ -0,0 +1,138 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
class AllModelsTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def test_true
|
6
|
+
true
|
7
|
+
end
|
8
|
+
|
9
|
+
# def test_map
|
10
|
+
# eachmeth = lambda do |profile|
|
11
|
+
# profile.claimed = false
|
12
|
+
# profile.save
|
13
|
+
# end
|
14
|
+
# data = [1,10,{:conditions => "profiles.claimed = 1"},"Profile",eachmeth]
|
15
|
+
#
|
16
|
+
# assert Profile.find(2).claimed, "claimed"
|
17
|
+
#
|
18
|
+
# ActiveRecord::Mapreduce.map([data])
|
19
|
+
#
|
20
|
+
# assert !Profile.find(2).claimed, "not claimed"
|
21
|
+
# end
|
22
|
+
#
|
23
|
+
# def test_each_with_proc
|
24
|
+
# Skynet.solo do
|
25
|
+
# ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 5).each do |profile|
|
26
|
+
# profile.suffix = 'z'
|
27
|
+
# profile.save
|
28
|
+
# end
|
29
|
+
# end
|
30
|
+
# assert_equal 'z', Profile.find(2).suffix
|
31
|
+
# end
|
32
|
+
#
|
33
|
+
# def test_big_batch
|
34
|
+
# Skynet.solo do
|
35
|
+
# ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 200, :model_class => "Profile").each do |profile|
|
36
|
+
# profile.suffix = 'z'
|
37
|
+
# profile.save
|
38
|
+
# end
|
39
|
+
# end
|
40
|
+
# assert_equal 'z', Profile.find(2).suffix
|
41
|
+
# end
|
42
|
+
#
|
43
|
+
# def test_joins
|
44
|
+
# p = Profile.find(1)
|
45
|
+
# p.details.zodiac_sign = 'hermit'
|
46
|
+
# p.details.save
|
47
|
+
# Skynet.solo do
|
48
|
+
# ActiveRecord::Mapreduce.find(:all, :conditions => "profile_details.zodiac_sign='hermit'", :joins => "JOIN profile_details ON profiles.id = profile_details.profile_id", :batch_size => 2, :model_class => "Profile").each do |profile|
|
49
|
+
# profile.suffix = 'z'
|
50
|
+
# profile.save
|
51
|
+
# end
|
52
|
+
# end
|
53
|
+
# assert_equal 'z', Profile.find(1).suffix
|
54
|
+
# end
|
55
|
+
#
|
56
|
+
# def test_stragglers
|
57
|
+
# profiles = Profile.find(:all)
|
58
|
+
# Skynet.solo do
|
59
|
+
# ActiveRecord::Mapreduce.find(:all, :batch_size => profiles.size-1, :model_class => "Profile").each do |profile|
|
60
|
+
# profile.suffix = 'z'
|
61
|
+
# profile.save
|
62
|
+
# end
|
63
|
+
# end
|
64
|
+
# profiles.last.reload
|
65
|
+
# assert_equal 'z', profiles.last.suffix
|
66
|
+
# end
|
67
|
+
#
|
68
|
+
# def test_small_limit
|
69
|
+
# Skynet.solo do
|
70
|
+
# ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 3).each do |profile|
|
71
|
+
# profile.suffix = 'z'
|
72
|
+
# profile.save
|
73
|
+
# end
|
74
|
+
# end
|
75
|
+
# assert_equal 'z', Profile.find(2).suffix
|
76
|
+
# end
|
77
|
+
#
|
78
|
+
# def test_each_with_proc_exception
|
79
|
+
# Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
|
80
|
+
# ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
|
81
|
+
# raise "BUSTED" if profile.id == 6
|
82
|
+
# profile.suffix = 'z'
|
83
|
+
# profile.save
|
84
|
+
# end
|
85
|
+
# end
|
86
|
+
# assert_equal 'z', Profile.find(2).suffix
|
87
|
+
# assert_equal nil, Profile.find(6).suffix
|
88
|
+
# end
|
89
|
+
#
|
90
|
+
# def test_each_with_class_exception
|
91
|
+
# Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
|
92
|
+
# ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(ActiveRecord::MapreduceExTest)
|
93
|
+
# end
|
94
|
+
# assert_equal 'k', Profile.find(2).suffix
|
95
|
+
# assert_equal nil, Profile.find(8).suffix
|
96
|
+
# end
|
97
|
+
#
|
98
|
+
# def test_each_with_class
|
99
|
+
# Skynet.solo do
|
100
|
+
# ActiveRecord::Mapreduce.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
|
101
|
+
# end
|
102
|
+
# assert_equal 'k', Profile.find(2).suffix
|
103
|
+
# end
|
104
|
+
#
|
105
|
+
# def test_distributed_each
|
106
|
+
# Skynet.solo do
|
107
|
+
# Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
|
108
|
+
# profile.suffix = 'gg'
|
109
|
+
# profile.save
|
110
|
+
# end
|
111
|
+
# assert_equal 'gg', Profile.find(2).suffix
|
112
|
+
# Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
|
113
|
+
# assert_equal 'k', Profile.find(2).suffix
|
114
|
+
# end
|
115
|
+
# end
|
116
|
+
#
|
117
|
+
# def test_distributed_each_with_symbol
|
118
|
+
# Skynet.solo do
|
119
|
+
# Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(:mark_modified)
|
120
|
+
# end
|
121
|
+
# assert ModifiedProfile.find(:first, :conditions => "profile_id = 2")
|
122
|
+
# end
|
123
|
+
#
|
124
|
+
# def self.each(profile)
|
125
|
+
# profile.suffix = 'k'
|
126
|
+
# profile.save
|
127
|
+
# end
|
128
|
+
|
129
|
+
end
|
130
|
+
|
131
|
+
class AllModelsExTest
|
132
|
+
def self.each(profile)
|
133
|
+
raise "BUSTED" if profile.id == 8
|
134
|
+
profile.suffix = 'k'
|
135
|
+
profile.save
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
data/test/test_helper.rb
CHANGED
@@ -1,12 +1,17 @@
|
|
1
|
-
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
2
|
|
3
|
-
require 'test/unit'
|
4
|
-
require '../lib/skynet.rb'
|
5
|
-
|
6
|
-
|
7
|
-
# Tests for the partitioners.
|
8
|
-
#
|
9
3
|
class MysqlMessageQueueTest < Test::Unit::TestCase
|
4
|
+
ENV["RAILS_ENV"] = "test"
|
5
|
+
|
6
|
+
ActiveRecord::Base.establish_connection(
|
7
|
+
:adapter => "mysql",
|
8
|
+
:host => "localhost",
|
9
|
+
:username => "root",
|
10
|
+
:password => "",
|
11
|
+
:database => "skynet_test"
|
12
|
+
)
|
13
|
+
ActiveRecord::Base.connection.execute("delete from skynet_message_queues")
|
14
|
+
ActiveRecord::Base.connection.execute("delete from skynet_worker_queues")
|
10
15
|
|
11
16
|
def setup
|
12
17
|
Skynet.configure(
|
@@ -15,24 +20,35 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
|
|
15
20
|
:SKYNET_LOG_FILE => STDOUT,
|
16
21
|
:SKYNET_LOG_LEVEL => Logger::ERROR,
|
17
22
|
:MESSAGE_QUEUE_ADAPTER => "Skynet::MessageQueueAdapter::Mysql",
|
18
|
-
:
|
23
|
+
:MYSQL_NEXT_TASK_TIMEOUT => 1,
|
24
|
+
:MYSQL_TEMPERATURE_CHANGE_SLEEP => 1
|
19
25
|
)
|
20
|
-
|
21
|
-
|
26
|
+
|
27
|
+
ActiveRecord::Base.establish_connection(
|
28
|
+
:adapter => "mysql",
|
29
|
+
:host => "localhost",
|
30
|
+
:username => "root",
|
31
|
+
:password => "",
|
32
|
+
:database => "skynet_test"
|
33
|
+
)
|
34
|
+
ActiveRecord::Base.connection.execute("delete from skynet_message_queues")
|
35
|
+
ActiveRecord::Base.connection.execute("delete from skynet_worker_queues")
|
22
36
|
|
23
|
-
@
|
24
|
-
:tasktype=>
|
25
|
-
:
|
26
|
-
:
|
27
|
-
:
|
28
|
-
:payload => "payload",
|
37
|
+
@message_options = {
|
38
|
+
:tasktype => "task",
|
39
|
+
:job_id => 1,
|
40
|
+
:task_id => 2,
|
41
|
+
:payload => "payload",
|
29
42
|
:payload_type => "task",
|
30
|
-
:expiry
|
31
|
-
:expire_time
|
32
|
-
:iteration
|
33
|
-
:name
|
34
|
-
:version
|
35
|
-
|
43
|
+
:expiry => 20,
|
44
|
+
:expire_time => 1095108406.9251,
|
45
|
+
:iteration => 0,
|
46
|
+
:name => "name",
|
47
|
+
:version => 1,
|
48
|
+
:queue_id => 0,
|
49
|
+
:retry => 1
|
50
|
+
}
|
51
|
+
@worker_message = Skynet::Message.new(@message_options)
|
36
52
|
|
37
53
|
end
|
38
54
|
|
@@ -40,8 +56,12 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
|
|
40
56
|
|
41
57
|
def test_write_message
|
42
58
|
mq.write_message(@worker_message,10)
|
43
|
-
message = SkynetMessageQueue.find(:
|
44
|
-
assert_equal @worker_message.expiry.to_f, 20
|
59
|
+
message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
|
60
|
+
assert_equal @worker_message.expiry.to_f, 20
|
61
|
+
@message_options.each do |key,val|
|
62
|
+
next if key == :payload
|
63
|
+
assert_equal message.send(key), val
|
64
|
+
end
|
45
65
|
end
|
46
66
|
|
47
67
|
def test_template_to_conditions
|
@@ -62,13 +82,16 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
|
|
62
82
|
end
|
63
83
|
|
64
84
|
def test_take_next_task
|
65
|
-
assert mq.write_message(@worker_message,10)
|
85
|
+
assert mq.write_message(@worker_message,10)
|
86
|
+
message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
|
87
|
+
assert_equal 0, message.iteration
|
66
88
|
task = mq.take_next_task(1,1,:task)
|
67
89
|
assert_equal @worker_message.payload, task.payload
|
68
|
-
|
69
|
-
message = SkynetMessageQueue.find(:first)
|
90
|
+
|
91
|
+
message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
|
70
92
|
assert_equal 1, message.iteration
|
71
93
|
assert @worker_message.expire_time < message.expire_time
|
94
|
+
|
72
95
|
excep = nil
|
73
96
|
begin
|
74
97
|
mq.take_next_task(1)
|
@@ -85,11 +108,36 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
|
|
85
108
|
task = mq.take_next_task(1)
|
86
109
|
assert_equal 2, task.task_id
|
87
110
|
assert_equal 0, task.iteration
|
88
|
-
sleep 0.6
|
89
111
|
ntt = Skynet::Message.next_task_template(1)
|
90
|
-
next_task = mq.take_next_task(1,
|
112
|
+
next_task = mq.take_next_task(1,1)
|
91
113
|
assert_equal 2, next_task.task_id
|
92
114
|
assert_equal 1, next_task.iteration
|
115
|
+
end
|
116
|
+
|
117
|
+
def test_take_race_condition
|
118
|
+
message = @worker_message.clone
|
119
|
+
message.expiry=0.4
|
120
|
+
assert mq.write_message(message)
|
121
|
+
template = Skynet::Message.next_task_template(1, :task, 0)
|
122
|
+
|
123
|
+
message_row = mq.send(:find_next_message,template,:task)
|
124
|
+
assert !message_row.tran_id
|
125
|
+
|
126
|
+
assert_equal 1, mq.send(:write_fallback_message,message_row, message)
|
127
|
+
assert_equal 0, mq.send(:write_fallback_message,message_row, message)
|
128
|
+
assert_equal 0, mq.send(:write_fallback_message,message_row, message)
|
129
|
+
|
130
|
+
template = Skynet::Message.next_task_template(1, :task, 0)
|
131
|
+
another_row = mq.send(:find_next_message,template,:task)
|
132
|
+
assert !another_row
|
133
|
+
sleep 1
|
134
|
+
template = Skynet::Message.next_task_template(1, :task, 0)
|
135
|
+
message_row2 = mq.send(:find_next_message,template,:task)
|
136
|
+
assert message_row2
|
137
|
+
assert_equal message_row.tran_id, message_row2.tran_id
|
138
|
+
|
139
|
+
message_row2.tran_id = 4
|
140
|
+
assert_equal 0, mq.send(:write_fallback_message,message_row2, message)
|
93
141
|
end
|
94
142
|
|
95
143
|
def test_write_and_take_result
|
@@ -187,8 +235,24 @@ class MysqlMessageQueueTest < Test::Unit::TestCase
|
|
187
235
|
assert_equal 10, mq.get_worker_version
|
188
236
|
mq.set_worker_version(11)
|
189
237
|
mq.set_worker_version(12)
|
190
|
-
assert_equal 1,
|
238
|
+
assert_equal 1, SkynetMessageQueue.count(:id, :conditions => "tasktype = 'version'")
|
191
239
|
end
|
240
|
+
|
241
|
+
def test_version_active?
|
242
|
+
mq.set_worker_version(98)
|
243
|
+
@worker_message.version = mq.get_worker_version
|
244
|
+
mq.write_message(@worker_message,10)
|
245
|
+
message = SkynetMessageQueue.find(:first, :conditions => "tasktype = 'task'")
|
246
|
+
assert_equal message.version, 98
|
247
|
+
mq.set_worker_version(99)
|
248
|
+
assert_equal 99, mq.get_worker_version
|
249
|
+
assert_equal true, mq.version_active?(98)
|
250
|
+
assert_equal true, mq.version_active?(99)
|
251
|
+
message.destroy
|
252
|
+
assert_equal false, mq.version_active?(98)
|
253
|
+
assert_equal true, mq.version_active?(99)
|
254
|
+
end
|
255
|
+
|
192
256
|
|
193
257
|
private
|
194
258
|
|
data/test/test_skynet.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
-
|
3
|
-
class TestSkynet < Test::Unit::TestCase
|
4
|
-
|
5
|
-
def setup
|
6
|
-
end
|
7
|
-
|
8
|
-
def test_truth
|
9
|
-
assert true
|
10
|
-
end
|
11
|
-
end
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
class TestSkynet < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_truth
|
9
|
+
assert true
|
10
|
+
end
|
11
|
+
end
|
@@ -28,17 +28,13 @@ class TestSkynetInstallGenerator < Test::Unit::TestCase
|
|
28
28
|
def test_generator_without_options
|
29
29
|
run_generator('skynet_install', [APP_ROOT], sources)
|
30
30
|
assert_directory_exists "script"
|
31
|
-
assert_directory_exists "db/migrate"
|
32
31
|
assert_generated_file "script/skynet"
|
33
|
-
assert_generated_file "script/skynet_console"
|
34
32
|
end
|
35
33
|
|
36
34
|
def test_generator_in_rails
|
37
35
|
run_generator('skynet_install', [APP_ROOT], sources, {"--rails"=>''})
|
38
36
|
assert_directory_exists "script"
|
39
|
-
assert_directory_exists "db/migrate"
|
40
37
|
assert_generated_file "script/skynet"
|
41
|
-
assert_generated_file "script/skynet_console"
|
42
38
|
end
|
43
39
|
|
44
40
|
private
|
@@ -0,0 +1,717 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
|
+
|
3
|
+
class SkynetJobTest < Test::Unit::TestCase
|
4
|
+
|
5
|
+
def setup
|
6
|
+
Skynet.configure(
|
7
|
+
:ENABLE => false,
|
8
|
+
:SKYNET_LOG_FILE => STDOUT,
|
9
|
+
:MESSAGE_QUEUE_ADAPTER => "Skynet::MessageQueueAdapter::TupleSpace",
|
10
|
+
:SKYNET_LOG_LEVEL => Logger::ERROR,
|
11
|
+
:TS_DRBURIS => ["druby://localhost:47999"],
|
12
|
+
:TS_USE_RINGSERVER => false
|
13
|
+
)
|
14
|
+
|
15
|
+
Skynet::MessageQueue.any_instance.stubs(:get_worker_version).returns(1)
|
16
|
+
Skynet::MessageQueue.any_instance.stubs(:set_worker_version).returns(1)
|
17
|
+
|
18
|
+
@ts = Rinda::TupleSpace.new
|
19
|
+
Skynet::MessageQueueAdapter::TupleSpace.stubs(:get_tuple_space).returns(@ts)
|
20
|
+
|
21
|
+
@messages = []
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_new_job
|
25
|
+
job = Skynet::Job.new(:map_reduce_class => JobMRTester)
|
26
|
+
{
|
27
|
+
:map_name => "JobMRTester MAP",
|
28
|
+
:map_timeout => 60,
|
29
|
+
:mappers => 2,
|
30
|
+
:name => "JobMRTester MASTER",
|
31
|
+
:reducers => 1,
|
32
|
+
:reduce_partition => nil,
|
33
|
+
:reduce => "JobMRTester",
|
34
|
+
:queue_id => 0,
|
35
|
+
:start_after => 0,
|
36
|
+
:reduce_timeout => 60,
|
37
|
+
:reduce_name => "JobMRTester REDUCE",
|
38
|
+
:master_timeout => 60,
|
39
|
+
:map => "JobMRTester",
|
40
|
+
:version => 1,
|
41
|
+
:result_timeout => 1200,
|
42
|
+
:master_result_timeout => 1200
|
43
|
+
}.each do |key, val|
|
44
|
+
assert_equal val, job.send(key), key
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_new_async_job
|
49
|
+
job = Skynet::AsyncJob.new(:map_reduce_class => JobMRTester)
|
50
|
+
{
|
51
|
+
:map_timeout => 60,
|
52
|
+
:reduce_partition => nil,
|
53
|
+
:master_timeout => 60,
|
54
|
+
:reduce_timeout => 60,
|
55
|
+
:name => "JobMRTester MASTER",
|
56
|
+
:version => 1,
|
57
|
+
:result_timeout => 1200,
|
58
|
+
:map_name => "JobMRTester MAP",
|
59
|
+
:reducers => 1,
|
60
|
+
:queue_id => 0,
|
61
|
+
:reduce_name => "JobMRTester REDUCE",
|
62
|
+
:map => "JobMRTester",
|
63
|
+
:async => true,
|
64
|
+
:master_result_timeout => 1200,
|
65
|
+
:mappers => 2,
|
66
|
+
:reduce => "JobMRTester",
|
67
|
+
:start_after => 0
|
68
|
+
}.each do |key, val|
|
69
|
+
assert_equal val, job.send(key), key
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def test_run_async
|
74
|
+
job = Skynet::AsyncJob.new(
|
75
|
+
:map_reduce_class => JobMRTester,
|
76
|
+
:version => 1,
|
77
|
+
:map_data => [[1]],
|
78
|
+
:mappers => 1
|
79
|
+
)
|
80
|
+
job.stubs(:use_local_queue?).returns(true)
|
81
|
+
assert_equal job.job_id, job.run
|
82
|
+
assert_equal 1, job.local_mq.messages.size
|
83
|
+
master_job = Skynet::Job.new(job.local_mq.messages.first.payload.process)
|
84
|
+
assert ! master_job.async
|
85
|
+
assert_equal true, master_job.local_master
|
86
|
+
end
|
87
|
+
|
88
|
+
def test_run_master
|
89
|
+
job = Skynet::Job.new(
|
90
|
+
:map_reduce_class => JobMRTester,
|
91
|
+
:version => 1,
|
92
|
+
:map_data => [[1]],
|
93
|
+
:mappers => 1,
|
94
|
+
:local_master => false
|
95
|
+
)
|
96
|
+
mq = functor
|
97
|
+
received_messages = []
|
98
|
+
mq.write_message = lambda do |message,timeout|
|
99
|
+
received_messages << message
|
100
|
+
end
|
101
|
+
mq.get_worker_version = 1
|
102
|
+
job.stubs(:mq).returns(mq)
|
103
|
+
result_message = functor(:payload => "result", :payload_type => :result, :task_id => 1)
|
104
|
+
mq.expects(:take_result).with(job.job_id,120).times(1).returns(result_message)
|
105
|
+
assert_equal ["result"], job.run
|
106
|
+
assert_equal 1, received_messages.size
|
107
|
+
end
|
108
|
+
|
109
|
+
def test_run
|
110
|
+
job = Skynet::Job.new(
|
111
|
+
:map_reduce_class => JobMRTester,
|
112
|
+
:version => 1,
|
113
|
+
:map_data => [1,2,3],
|
114
|
+
:mappers => 2,
|
115
|
+
:reducers => 2
|
116
|
+
)
|
117
|
+
job.stubs(:use_local_queue?).returns(true)
|
118
|
+
results = job.run
|
119
|
+
assert_equal 2, results.size
|
120
|
+
assert_equal [1,2,3], results.flatten.sort
|
121
|
+
end
|
122
|
+
|
123
|
+
def test_master_enqueue
|
124
|
+
job = Skynet::AsyncJob.new(
|
125
|
+
:map_reduce_class => JobMRTester,
|
126
|
+
:version => 1,
|
127
|
+
:map_data => [[1]],
|
128
|
+
:mappers => 1
|
129
|
+
)
|
130
|
+
|
131
|
+
mq = functor
|
132
|
+
job.stubs(:mq).returns(mq)
|
133
|
+
received_messages = []
|
134
|
+
mq.write_message = lambda do |message,timeout|
|
135
|
+
received_messages << message
|
136
|
+
end
|
137
|
+
mq.get_worker_version = 1
|
138
|
+
|
139
|
+
job.master_enqueue
|
140
|
+
assert_equal :master, received_messages.first.payload_type
|
141
|
+
assert_equal :master, received_messages.first.payload.map_or_reduce
|
142
|
+
master_job = Skynet::Job.new(received_messages.first.payload.process)
|
143
|
+
assert_equal JobMRTester.to_s, master_job.map
|
144
|
+
assert ! master_job.async
|
145
|
+
assert_equal true, master_job.local_master
|
146
|
+
end
|
147
|
+
|
148
|
+
def test_master_results
|
149
|
+
job = Skynet::Job.new(
|
150
|
+
:map_reduce_class => JobMRTester,
|
151
|
+
:version => 1,
|
152
|
+
:map_data => [[1]],
|
153
|
+
:mappers => 1
|
154
|
+
)
|
155
|
+
|
156
|
+
mq = functor(:payload => "result")
|
157
|
+
result_message = functor(:payload => "result", :payload_type => :result, :task_id => 1)
|
158
|
+
mq.expects(:take_result).with(job.job_id,120).times(1).returns(result_message)
|
159
|
+
job.stubs(:mq).returns(mq)
|
160
|
+
results = job.master_results
|
161
|
+
assert_equal "result", results.first
|
162
|
+
end
|
163
|
+
|
164
|
+
def test_map_enqueue
|
165
|
+
job = Skynet::AsyncJob.new(
|
166
|
+
:map_reduce_class => JobMRTester,
|
167
|
+
:version => 1,
|
168
|
+
:map_data => [1,2,3],
|
169
|
+
:mappers => 2
|
170
|
+
)
|
171
|
+
|
172
|
+
mq = functor
|
173
|
+
job.stubs(:mq).returns(mq)
|
174
|
+
received_messages = []
|
175
|
+
mq.write_message = lambda do |message,timeout|
|
176
|
+
received_messages << message
|
177
|
+
end
|
178
|
+
mq.get_worker_version = 1
|
179
|
+
|
180
|
+
job.map_enqueue
|
181
|
+
assert_equal 2, received_messages.size
|
182
|
+
assert_equal :task, received_messages.first.payload_type
|
183
|
+
assert_equal :map, received_messages.first.payload.map_or_reduce
|
184
|
+
end
|
185
|
+
|
186
|
+
def test_map_results
|
187
|
+
job = Skynet::AsyncJob.new(
|
188
|
+
:map_reduce_class => JobMRTester,
|
189
|
+
:version => 1,
|
190
|
+
:map_data => [[1]],
|
191
|
+
:mappers => 1
|
192
|
+
)
|
193
|
+
|
194
|
+
mq = functor(:payload => "result")
|
195
|
+
i = 0
|
196
|
+
result_message = functor(:payload => "result", :payload_type => :result, :task_id => lambda {i += 1})
|
197
|
+
mq.expects(:take_result).with(job.job_id,120).times(2).returns(result_message)
|
198
|
+
job.stubs(:mq).returns(mq)
|
199
|
+
results = job.map_results(2)
|
200
|
+
assert_equal "result", results.first
|
201
|
+
end
|
202
|
+
|
203
|
+
def test_local_map_results
|
204
|
+
job = Skynet::AsyncJob.new(
|
205
|
+
:map_reduce_class => JobMRTester,
|
206
|
+
:version => 1,
|
207
|
+
:map_data => [1,2],
|
208
|
+
:mappers => 2,
|
209
|
+
:single => true
|
210
|
+
)
|
211
|
+
|
212
|
+
job.map_enqueue
|
213
|
+
assert_equal 2, job.local_mq.messages.size
|
214
|
+
results = job.map_results(2)
|
215
|
+
assert_equal [[1],[2]], results.sort
|
216
|
+
end
|
217
|
+
|
218
|
+
def test_partition_data
|
219
|
+
job = Skynet::AsyncJob.new(
|
220
|
+
:map_reduce_class => JobMRTester,
|
221
|
+
:version => 1,
|
222
|
+
:map_data => [[1]],
|
223
|
+
:reducers => 2
|
224
|
+
)
|
225
|
+
|
226
|
+
partitioned_data = job.partition_data([[1],[2],[3]])
|
227
|
+
assert_equal [[1, 3], [2]], partitioned_data
|
228
|
+
end
|
229
|
+
|
230
|
+
def test_partition_data_class
|
231
|
+
job = Skynet::AsyncJob.new(
|
232
|
+
:map_reduce_class => "JobPartitionTest",
|
233
|
+
:version => 1,
|
234
|
+
:map_data => [[1]],
|
235
|
+
:reducers => 2
|
236
|
+
)
|
237
|
+
|
238
|
+
partitioned_data = job.partition_data([[1],[2],[3]])
|
239
|
+
assert_equal [[1], [2], [3]], partitioned_data
|
240
|
+
end
|
241
|
+
|
242
|
+
def test_reduce_enqueue
|
243
|
+
job = Skynet::AsyncJob.new(
|
244
|
+
:map_reduce_class => JobMRTester,
|
245
|
+
:version => 1,
|
246
|
+
:map_data => [1,2,3],
|
247
|
+
:mappers => 2
|
248
|
+
)
|
249
|
+
|
250
|
+
mq = functor
|
251
|
+
job.stubs(:mq).returns(mq)
|
252
|
+
received_messages = []
|
253
|
+
mq.write_message = lambda do |message,timeout|
|
254
|
+
received_messages << message
|
255
|
+
end
|
256
|
+
mq.get_worker_version = 1
|
257
|
+
|
258
|
+
job.reduce_enqueue([[1, 3], [2]])
|
259
|
+
assert_equal 2, received_messages.size
|
260
|
+
assert_equal :task, received_messages.first.payload_type
|
261
|
+
assert_equal :reduce, received_messages.first.payload.map_or_reduce
|
262
|
+
end
|
263
|
+
|
264
|
+
def test_reduce_results
|
265
|
+
job = Skynet::AsyncJob.new(
|
266
|
+
:map_reduce_class => JobMRTester,
|
267
|
+
:version => 1,
|
268
|
+
:map_data => [[1]],
|
269
|
+
:mappers => 1
|
270
|
+
)
|
271
|
+
|
272
|
+
mq = functor(:payload => "result")
|
273
|
+
i = 0
|
274
|
+
result_message = functor(:payload => "result", :payload_type => :result, :task_id => lambda {i += 1})
|
275
|
+
mq.expects(:take_result).with(job.job_id,120).times(2).returns(result_message)
|
276
|
+
job.stubs(:mq).returns(mq)
|
277
|
+
results = job.reduce_results(2)
|
278
|
+
assert_equal "result", results.first
|
279
|
+
end
|
280
|
+
|
281
|
+
def test_master_task
|
282
|
+
job = Skynet::AsyncJob.new(:map_reduce_class => JobMRTester,:version=>1, :queue_id => 4)
|
283
|
+
mt = job.master_task
|
284
|
+
assert mt.is_a?(Skynet::Task)
|
285
|
+
assert_equal mt.result_timeout, 60
|
286
|
+
master_job = Skynet::Job.new(mt.process)
|
287
|
+
assert_equal job.map, JobMRTester.to_s
|
288
|
+
assert_equal job.reduce, JobMRTester.to_s
|
289
|
+
assert_equal nil, job.reduce_partition
|
290
|
+
assert_equal master_job.map, JobMRTester.to_s
|
291
|
+
assert_equal master_job.reduce, JobMRTester.to_s
|
292
|
+
assert_equal nil, master_job.reduce_partition
|
293
|
+
assert_equal 4, master_job.queue_id
|
294
|
+
Skynet::Job::FIELDS.each do |field|
|
295
|
+
case field
|
296
|
+
when :async, :local_master
|
297
|
+
nil
|
298
|
+
when :job_id, :single
|
299
|
+
next
|
300
|
+
else
|
301
|
+
assert_equal job.send(field), master_job.send(field), "Testing #{field}, jobfield: #{job.send(field)} mjobfield: #{master_job.send(field)}"
|
302
|
+
end
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
def test_gather_results_with_errors
|
307
|
+
job = Skynet::AsyncJob.new(
|
308
|
+
:map_reduce_class => JobMRTester,
|
309
|
+
:version => 1,
|
310
|
+
:map_data => [1],
|
311
|
+
:mappers => 1
|
312
|
+
)
|
313
|
+
map_tasks = job.map_tasks
|
314
|
+
mq = mq
|
315
|
+
mq.stubs(:get_worker_version).returns(1)
|
316
|
+
job.stubs(:mq).returns(mq)
|
317
|
+
messages = job.tasks_to_messages(map_tasks)
|
318
|
+
message = messages.first.result_message(["works"])
|
319
|
+
message2 = Skynet::Message.new(message.to_h.merge(:payload_type => :error, :payload => "error", :task_id => 33))
|
320
|
+
|
321
|
+
test_message1 = {
|
322
|
+
:version =>1,
|
323
|
+
:queue_id =>0,
|
324
|
+
:iteration =>0,
|
325
|
+
:name =>"JobMRTester MAP",
|
326
|
+
:tasktype =>:result,
|
327
|
+
:expire_time =>0,
|
328
|
+
:payload_type =>:result,
|
329
|
+
:payload =>["works"],
|
330
|
+
:drburi =>nil,
|
331
|
+
:expiry =>60,
|
332
|
+
:retry =>3
|
333
|
+
}
|
334
|
+
assert message.task_id.is_a?(Bignum)
|
335
|
+
assert message.job_id.is_a?(Bignum)
|
336
|
+
test_message1.each do |k,v|
|
337
|
+
assert_equal v, message.send(k), k
|
338
|
+
end
|
339
|
+
|
340
|
+
mq.expects(:take_result).with(job.job_id, 120).returns(message,message2).times(2)
|
341
|
+
results = nil
|
342
|
+
Skynet.silent do
|
343
|
+
results = job.gather_results(2, map_tasks.first.result_timeout, map_tasks.first.name)
|
344
|
+
end
|
345
|
+
assert_equal [["works"]], results
|
346
|
+
end
|
347
|
+
|
348
|
+
def test_map_tasks
|
349
|
+
job = Skynet::AsyncJob.new(
|
350
|
+
:map_reduce_class => JobMRTester,
|
351
|
+
:version => 1,
|
352
|
+
:map_data => [1,2,3],
|
353
|
+
:map_retry => 7,
|
354
|
+
:mappers => 2
|
355
|
+
)
|
356
|
+
map_tasks = job.map_tasks
|
357
|
+
assert_equal 2, map_tasks.size
|
358
|
+
assert_equal 7, map_tasks.first.retry
|
359
|
+
assert map_tasks[0].task_id != map_tasks[1].task_id
|
360
|
+
end
|
361
|
+
|
362
|
+
def test_reduce_tasks
|
363
|
+
job = Skynet::AsyncJob.new(
|
364
|
+
:map_reduce_class => JobMRTester,
|
365
|
+
:version => 1,
|
366
|
+
:map_data => [1,2,3],
|
367
|
+
:reduce_retry => 9,
|
368
|
+
:reducers => 2
|
369
|
+
)
|
370
|
+
reduce_tasks = job.reduce_tasks([1,2,3])
|
371
|
+
assert_equal 3, reduce_tasks.size
|
372
|
+
assert_equal 9, reduce_tasks.first.retry
|
373
|
+
assert reduce_tasks[0].task_id != reduce_tasks[1].task_id
|
374
|
+
end
|
375
|
+
|
376
|
+
def test_enqueue_messages
|
377
|
+
passed_messages = []
|
378
|
+
mq = functor
|
379
|
+
mq.write_message = lambda do |m,timeout|
|
380
|
+
passed_messages << m
|
381
|
+
end
|
382
|
+
job = Skynet::Job.new(:map_data => [1,2,3], :map_reduce_class => JobMRTester)
|
383
|
+
job.stubs(:mq).returns(mq)
|
384
|
+
message1 = Skynet::Message.new(
|
385
|
+
:tasktype => :task,
|
386
|
+
:task_id => 9,
|
387
|
+
:job_id => 8,
|
388
|
+
:payload_type => :task,
|
389
|
+
:payload => "blah",
|
390
|
+
:retry => 3,
|
391
|
+
:iteration => 0,
|
392
|
+
:version => 1,
|
393
|
+
:queue_id => 4,
|
394
|
+
:name => "test"
|
395
|
+
)
|
396
|
+
message2 = Skynet::Message.new(message1.to_h.merge(:name => "test2", :payload => "test2"))
|
397
|
+
job.enqueue_messages([message1,message2])
|
398
|
+
assert_equal [message1, message2], passed_messages
|
399
|
+
end
|
400
|
+
|
401
|
+
|
402
|
+
def test_local_queue_write_message
|
403
|
+
local_queue = Skynet::Job::LocalMessageQueue.new
|
404
|
+
assert_equal 1, local_queue.get_worker_version
|
405
|
+
local_queue.write_message(Skynet::Message.new({}),2)
|
406
|
+
assert local_queue.messages.first.is_a?(Skynet::Message)
|
407
|
+
end
|
408
|
+
|
409
|
+
def test_local_queue_take_result
|
410
|
+
job = Skynet::AsyncJob.new(
|
411
|
+
:map_reduce_class => JobMRTester,
|
412
|
+
:queue_id => 6,
|
413
|
+
:version => 1,
|
414
|
+
:map_data => [1,2,3],
|
415
|
+
:mappers => 2,
|
416
|
+
:master_retry => 17,
|
417
|
+
:master_result_timeout => 1
|
418
|
+
)
|
419
|
+
tasks = job.map_tasks
|
420
|
+
assert_equal 2, tasks.size
|
421
|
+
messages = job.tasks_to_messages(tasks)
|
422
|
+
|
423
|
+
local_queue = Skynet::Job::LocalMessageQueue.new
|
424
|
+
messages.each do |message|
|
425
|
+
local_queue.write_message(message)
|
426
|
+
end
|
427
|
+
assert_equal messages, local_queue.messages
|
428
|
+
assert_equal [1,3], local_queue.take_result(job.job_id,2).payload
|
429
|
+
assert_equal [2], local_queue.take_result(job.job_id,2).payload
|
430
|
+
assert_equal 0, local_queue.messages.size
|
431
|
+
assert_equal 0, local_queue.results.size
|
432
|
+
end
|
433
|
+
|
434
|
+
def test_run_tasks_locally_errors
|
435
|
+
job = Skynet::AsyncJob.new(
|
436
|
+
:map_reduce_class => JobMRTester,
|
437
|
+
:queue_id => 6,
|
438
|
+
:version => 1,
|
439
|
+
:map_data => [1,2,3],
|
440
|
+
:mappers => 2,
|
441
|
+
:master_retry => 17,
|
442
|
+
:master_result_timeout => 1
|
443
|
+
)
|
444
|
+
tasks = job.map_tasks
|
445
|
+
assert_equal 2, tasks.size
|
446
|
+
|
447
|
+
messages = job.tasks_to_messages(tasks)
|
448
|
+
|
449
|
+
task1 = messages.first.payload
|
450
|
+
task1.extend(Functor)
|
451
|
+
tries = 0
|
452
|
+
task1.define_method(:tries) {@tries}
|
453
|
+
task1.define_method(:run) do
|
454
|
+
@tries ||= 0
|
455
|
+
@tries += 1
|
456
|
+
if @tries == 1
|
457
|
+
raise Exception
|
458
|
+
else
|
459
|
+
return task1.data
|
460
|
+
end
|
461
|
+
end
|
462
|
+
messages.first.expects(:payload).returns(task1).times(2)
|
463
|
+
|
464
|
+
task2 = messages[1].payload
|
465
|
+
task2.extend(Functor)
|
466
|
+
tries = 0
|
467
|
+
task2.define_method(:tries) {@tries}
|
468
|
+
task2.define_method(:run) do
|
469
|
+
@tries ||= 0
|
470
|
+
@tries += 1
|
471
|
+
return task2.data
|
472
|
+
end
|
473
|
+
messages[1].expects(:payload).returns(task2).times(1)
|
474
|
+
|
475
|
+
local_queue = Skynet::Job::LocalMessageQueue.new
|
476
|
+
messages.each do |message|
|
477
|
+
local_queue.write_message(message)
|
478
|
+
end
|
479
|
+
|
480
|
+
assert_equal messages, local_queue.messages
|
481
|
+
Skynet.configure(:SKYNET_LOG_LEVEL=>Logger::FATAL) do
|
482
|
+
assert_equal [1,3], local_queue.take_result(job.job_id,2).payload
|
483
|
+
assert_equal [2], local_queue.take_result(job.job_id,2).payload
|
484
|
+
end
|
485
|
+
assert_equal 2, task1.tries
|
486
|
+
assert_equal 1, task2.tries
|
487
|
+
assert_equal 0, local_queue.messages.size
|
488
|
+
assert_equal 0, local_queue.results.size
|
489
|
+
end
|
490
|
+
|
491
|
+
def test_map_local
|
492
|
+
job = Skynet::AsyncJob.new(
|
493
|
+
:map_reduce_class => JobMRTester,
|
494
|
+
:version => 1,
|
495
|
+
:map_data => [1,2,3,4],
|
496
|
+
:mappers => 3
|
497
|
+
)
|
498
|
+
|
499
|
+
assert_equal 3, job.map_tasks.size
|
500
|
+
assert !job.map_local?
|
501
|
+
|
502
|
+
job.keep_map_tasks = true
|
503
|
+
assert job.map_local?
|
504
|
+
|
505
|
+
job.keep_map_tasks = 3
|
506
|
+
assert job.map_local?
|
507
|
+
|
508
|
+
job.keep_map_tasks = 2
|
509
|
+
assert !job.map_local?
|
510
|
+
|
511
|
+
job.single = true
|
512
|
+
assert job.map_local?
|
513
|
+
end
|
514
|
+
|
515
|
+
def test_reduce_local
|
516
|
+
job = Skynet::AsyncJob.new(
|
517
|
+
:reduce_reduce_class => JobMRTester,
|
518
|
+
:version => 1,
|
519
|
+
:map_data => [1,2,3,4],
|
520
|
+
:reducers => 3
|
521
|
+
)
|
522
|
+
|
523
|
+
partitioned_data = [[1,2],[3],[4]]
|
524
|
+
assert_equal 3, job.reduce_tasks(partitioned_data).size
|
525
|
+
assert !job.reduce_local?(job.reduce_tasks(partitioned_data))
|
526
|
+
|
527
|
+
job.keep_reduce_tasks = true
|
528
|
+
assert job.reduce_local?(job.reduce_tasks(partitioned_data))
|
529
|
+
|
530
|
+
job.keep_reduce_tasks = 3
|
531
|
+
assert job.reduce_local?(job.reduce_tasks(partitioned_data))
|
532
|
+
|
533
|
+
job.keep_reduce_tasks = 2
|
534
|
+
assert !job.reduce_local?(job.reduce_tasks(partitioned_data))
|
535
|
+
|
536
|
+
job.single = true
|
537
|
+
assert job.reduce_local?(job.reduce_tasks(partitioned_data))
|
538
|
+
end
|
539
|
+
|
540
|
+
|
541
|
+
|
542
|
+
def test_keep_map_tasks
|
543
|
+
job = Skynet::Job.new(
|
544
|
+
:map_reduce_class => JobMRTester,
|
545
|
+
:version => 1,
|
546
|
+
:map_data => [1,2],
|
547
|
+
:mappers => 2,
|
548
|
+
:keep_map_tasks => 2
|
549
|
+
)
|
550
|
+
assert_equal 2, job.map_enqueue
|
551
|
+
assert_equal 2, job.local_mq.messages.size
|
552
|
+
assert_equal [[1],[2]], job.map_results(2).sort
|
553
|
+
assert_equal 0, job.local_mq.messages.size
|
554
|
+
assert_equal 0, job.local_mq.results.size
|
555
|
+
end
|
556
|
+
|
557
|
+
|
558
|
+
def test_keep_reduce_tasks
|
559
|
+
job = Skynet::Job.new(
|
560
|
+
:map_reduce_class => JobMRTester,
|
561
|
+
:version => 1,
|
562
|
+
:map_data => [1,2],
|
563
|
+
:reducers => 1,
|
564
|
+
:keep_reduce_tasks => 2
|
565
|
+
)
|
566
|
+
assert_equal 1, job.reduce_enqueue([[1,2]])
|
567
|
+
assert_equal 1, job.local_mq.messages.size
|
568
|
+
assert_equal [[1,2]], job.reduce_results(1).sort
|
569
|
+
assert_equal 0, job.local_mq.messages.size
|
570
|
+
assert_equal 0, job.local_mq.results.size
|
571
|
+
end
|
572
|
+
|
573
|
+
def test_mapreduce_helper_mixin
|
574
|
+
job = Skynet::Job.new(
|
575
|
+
:map_reduce_class => "JobMapreduceHelperTest",
|
576
|
+
:version => 1,
|
577
|
+
:map_data => [1,2],
|
578
|
+
:reducers => 2,
|
579
|
+
:keep_reduce_tasks => true,
|
580
|
+
:keep_map_tasks => true
|
581
|
+
)
|
582
|
+
map_results = nil
|
583
|
+
results = nil
|
584
|
+
Skynet.solo do
|
585
|
+
map_results = job.map_results(job.map_enqueue)
|
586
|
+
partitioned_data = job.partition_data(map_results)
|
587
|
+
results = job.reduce_results(job.reduce_enqueue(partitioned_data))
|
588
|
+
end
|
589
|
+
assert_equal [2,3], map_results.flatten.sort
|
590
|
+
assert_equal [3,4], results.flatten.sort
|
591
|
+
end
|
592
|
+
|
593
|
+
def test_mapreduce_helper_mixin_again
|
594
|
+
[JobMRTest2, JobMRTest3].each do |klass|
|
595
|
+
job = Skynet::Job.new(
|
596
|
+
:mappers => 2,
|
597
|
+
:reducers => 1,
|
598
|
+
:map_reduce_class => klass,
|
599
|
+
:map_data => [
|
600
|
+
OpenStruct.new({:created_by => 2}),
|
601
|
+
OpenStruct.new({:created_by => 2}),
|
602
|
+
OpenStruct.new({:created_by => 3})]
|
603
|
+
)
|
604
|
+
|
605
|
+
map_results = nil
|
606
|
+
results = nil
|
607
|
+
Skynet.solo do
|
608
|
+
map_results = job.map_results(job.map_enqueue)
|
609
|
+
partitioned_data = job.partition_data(map_results)
|
610
|
+
results = job.reduce_results(job.reduce_enqueue(partitioned_data))
|
611
|
+
end
|
612
|
+
assert_equal [1, 1, 1, 2, 2, 3], map_results.flatten.sort
|
613
|
+
expected_results = {2=>2, 3=>1}
|
614
|
+
assert_equal expected_results, results
|
615
|
+
end
|
616
|
+
end
|
617
|
+
|
618
|
+
|
619
|
+
private
|
620
|
+
|
621
|
+
def mq
|
622
|
+
Skynet::MessageQueueAdapter::TupleSpace.new
|
623
|
+
end
|
624
|
+
|
625
|
+
end
|
626
|
+
|
627
|
+
|
628
|
+
class JobMRTester
|
629
|
+
def self.map(datas)
|
630
|
+
ret = []
|
631
|
+
datas.each do |data|
|
632
|
+
if data == :error
|
633
|
+
raise Exception.new("something bad happened")
|
634
|
+
else
|
635
|
+
ret << data
|
636
|
+
end
|
637
|
+
end
|
638
|
+
return ret
|
639
|
+
end
|
640
|
+
|
641
|
+
def self.reduce(datas)
|
642
|
+
datas
|
643
|
+
end
|
644
|
+
end
|
645
|
+
|
646
|
+
class JobPartitionTest < JobMRTester
|
647
|
+
|
648
|
+
def self.reduce_partition(post_map_data, reducers)
|
649
|
+
return post_map_data.compact
|
650
|
+
end
|
651
|
+
end
|
652
|
+
|
653
|
+
class JobMapreduceHelperTest
|
654
|
+
include MapreduceHelper
|
655
|
+
|
656
|
+
def self.map_each(data)
|
657
|
+
return data + 1
|
658
|
+
end
|
659
|
+
|
660
|
+
def self.reduce_each(data)
|
661
|
+
return data + 1
|
662
|
+
end
|
663
|
+
end
|
664
|
+
|
665
|
+
class JobMRTest2
|
666
|
+
include MapreduceHelper
|
667
|
+
|
668
|
+
# def self.map(profiles)
|
669
|
+
# result = Array.new
|
670
|
+
# profiles.each do |profile|
|
671
|
+
# result << [profile.created_by, 1] if profile.created_by
|
672
|
+
# end
|
673
|
+
# result
|
674
|
+
# end
|
675
|
+
|
676
|
+
def self.map_each(item)
|
677
|
+
return [item.created_by, 1] if item.created_by
|
678
|
+
end
|
679
|
+
|
680
|
+
def self.reduce(pairs)
|
681
|
+
totals = Hash.new
|
682
|
+
pairs.each do |pair|
|
683
|
+
created_by, count = pair[0], pair[1]
|
684
|
+
totals[created_by] ||= 0
|
685
|
+
totals[created_by] += count
|
686
|
+
end
|
687
|
+
return totals
|
688
|
+
end
|
689
|
+
end
|
690
|
+
|
691
|
+
class JobMRTest3
|
692
|
+
|
693
|
+
include MapreduceHelper
|
694
|
+
|
695
|
+
def self.map(profiles)
|
696
|
+
result = Array.new
|
697
|
+
profiles.each do |profile|
|
698
|
+
result << [profile.created_by, 1] if profile.created_by
|
699
|
+
end
|
700
|
+
result
|
701
|
+
end
|
702
|
+
|
703
|
+
# def self.map_each(item)
|
704
|
+
# return [item.created_by, 1] if item.created_by
|
705
|
+
# end
|
706
|
+
|
707
|
+
def self.reduce(pairs)
|
708
|
+
totals = Hash.new
|
709
|
+
pairs.each do |pair|
|
710
|
+
created_by, count = pair[0], pair[1]
|
711
|
+
totals[created_by] ||= 0
|
712
|
+
totals[created_by] += count
|
713
|
+
end
|
714
|
+
|
715
|
+
return totals
|
716
|
+
end
|
717
|
+
end
|