skynet 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +4 -0
- data/License.txt +20 -0
- data/Manifest.txt +65 -0
- data/README.txt +100 -0
- data/Rakefile +4 -0
- data/app_generators/skynet_install/USAGE +5 -0
- data/app_generators/skynet_install/skynet_install_generator.rb +84 -0
- data/app_generators/skynet_install/templates/migration.rb +60 -0
- data/app_generators/skynet_install/templates/skynet +33 -0
- data/app_generators/skynet_install/templates/skynet_console +16 -0
- data/bin/skynet +20 -0
- data/bin/skynet_console +9 -0
- data/bin/skynet_install +12 -0
- data/bin/skynet_tuplespace_server +53 -0
- data/config/hoe.rb +74 -0
- data/config/requirements.rb +17 -0
- data/lib/skynet.rb +34 -0
- data/lib/skynet/mapreduce_test.rb +25 -0
- data/lib/skynet/message_queue_adapters/message_queue_adapter.rb +70 -0
- data/lib/skynet/message_queue_adapters/mysql.rb +573 -0
- data/lib/skynet/message_queue_adapters/tuple_space.rb +327 -0
- data/lib/skynet/skynet_active_record_extensions.rb +237 -0
- data/lib/skynet/skynet_config.rb +59 -0
- data/lib/skynet/skynet_console.rb +34 -0
- data/lib/skynet/skynet_console_helper.rb +59 -0
- data/lib/skynet/skynet_debugger.rb +84 -0
- data/lib/skynet/skynet_guid_generator.rb +68 -0
- data/lib/skynet/skynet_job.rb +607 -0
- data/lib/skynet/skynet_launcher.rb +10 -0
- data/lib/skynet/skynet_logger.rb +52 -0
- data/lib/skynet/skynet_manager.rb +486 -0
- data/lib/skynet/skynet_message.rb +366 -0
- data/lib/skynet/skynet_message_queue.rb +100 -0
- data/lib/skynet/skynet_ruby_extensions.rb +36 -0
- data/lib/skynet/skynet_task.rb +76 -0
- data/lib/skynet/skynet_tuplespace_server.rb +82 -0
- data/lib/skynet/skynet_worker.rb +395 -0
- data/lib/skynet/version.rb +9 -0
- data/log/debug.log +0 -0
- data/log/skynet.log +29 -0
- data/log/skynet_tuplespace_server.log +7 -0
- data/log/skynet_worker.pid +1 -0
- data/script/destroy +14 -0
- data/script/generate +14 -0
- data/script/txt2html +74 -0
- data/setup.rb +1585 -0
- data/sometest.rb +23 -0
- data/tasks/deployment.rake +34 -0
- data/tasks/environment.rake +7 -0
- data/tasks/website.rake +17 -0
- data/test/all_models_test.rb +139 -0
- data/test/mysql_message_queue_adaptor_test.rb +199 -0
- data/test/skynet_manager_test.rb +107 -0
- data/test/skynet_message_test.rb +42 -0
- data/test/test_generator_helper.rb +20 -0
- data/test/test_helper.rb +2 -0
- data/test/test_skynet.rb +11 -0
- data/test/test_skynet_install_generator.rb +53 -0
- data/test/tuplespace_message_queue_test.rb +179 -0
- data/tmtags +1242 -0
- data/website/index.html +93 -0
- data/website/index.txt +39 -0
- data/website/javascripts/rounded_corners_lite.inc.js +285 -0
- data/website/stylesheets/screen.css +138 -0
- data/website/template.rhtml +48 -0
- metadata +129 -0
data/sometest.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
class SomeTest
|
2
|
+
include SkynetDebugger
|
3
|
+
|
4
|
+
def self.map(datas)
|
5
|
+
results = {}
|
6
|
+
datas.each do |data|
|
7
|
+
results[data] ||= 0
|
8
|
+
results[data] += 1
|
9
|
+
end
|
10
|
+
[results]
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.reduce(datas)
|
14
|
+
results = {}
|
15
|
+
datas.each do |hashes|
|
16
|
+
hashes.each do |key,value|
|
17
|
+
results[key] ||= 0
|
18
|
+
results[key] += value
|
19
|
+
end
|
20
|
+
end
|
21
|
+
results
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
desc 'Release the website and new gem version'
|
2
|
+
task :deploy => [:check_version, :website, :release] do
|
3
|
+
puts "Remember to create SVN tag:"
|
4
|
+
puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
|
5
|
+
"svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
|
6
|
+
puts "Suggested comment:"
|
7
|
+
puts "Tagging release #{CHANGES}"
|
8
|
+
end
|
9
|
+
|
10
|
+
desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
|
11
|
+
task :local_deploy => [:website_generate, :install_gem]
|
12
|
+
|
13
|
+
task :check_version do
|
14
|
+
unless ENV['VERSION']
|
15
|
+
puts 'Must pass a VERSION=x.y.z release version'
|
16
|
+
exit
|
17
|
+
end
|
18
|
+
unless ENV['VERSION'] == VERS
|
19
|
+
puts "Please update your version.rb to match the release version, currently #{VERS}"
|
20
|
+
exit
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
desc 'Install the package as a gem, without generating documentation(ri/rdoc)'
|
25
|
+
task :install_gem_no_doc => [:clean, :package] do
|
26
|
+
sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri"
|
27
|
+
end
|
28
|
+
|
29
|
+
namespace :manifest do
|
30
|
+
desc 'Recreate Manifest.txt to include ALL files'
|
31
|
+
task :refresh do
|
32
|
+
`rake check_manifest | patch -p0 > Manifest.txt`
|
33
|
+
end
|
34
|
+
end
|
data/tasks/website.rake
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
desc 'Generate website files'
|
2
|
+
task :website_generate => :ruby_env do
|
3
|
+
(Dir['website/**/*.txt'] - Dir['website/version*.txt']).each do |txt|
|
4
|
+
sh %{ #{RUBY_APP} script/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
|
5
|
+
end
|
6
|
+
end
|
7
|
+
|
8
|
+
desc 'Upload website files to rubyforge'
|
9
|
+
task :website_upload do
|
10
|
+
host = "#{rubyforge_username}@rubyforge.org"
|
11
|
+
remote_dir = "/var/www/gforge-projects/#{PATH}/"
|
12
|
+
local_dir = 'website'
|
13
|
+
sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
|
14
|
+
end
|
15
|
+
|
16
|
+
desc 'Generate and upload website files'
|
17
|
+
task :website => [:website_generate, :website_upload, :publish_docs]
|
@@ -0,0 +1,139 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../../functional_test_helper'
|
2
|
+
require 'mapreduce_lib/all_models'
|
3
|
+
|
4
|
+
module MapreduceLib
|
5
|
+
class AllModelsTest < Test::Unit::TestCase
|
6
|
+
|
7
|
+
fixtures :profiles, :profile_details
|
8
|
+
|
9
|
+
def test_map
|
10
|
+
eachmeth = lambda do |profile|
|
11
|
+
profile.claimed = false
|
12
|
+
profile.save
|
13
|
+
end
|
14
|
+
data = [1,10,{:conditions => "profiles.claimed = 1"},"Profile",eachmeth]
|
15
|
+
|
16
|
+
assert Profile.find(2).claimed, "claimed"
|
17
|
+
|
18
|
+
MapreduceLib::AllModels.map([data])
|
19
|
+
|
20
|
+
assert !Profile.find(2).claimed, "not claimed"
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_each_with_proc
|
24
|
+
Skynet.solo do
|
25
|
+
MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 5).each do |profile|
|
26
|
+
profile.suffix = 'z'
|
27
|
+
profile.save
|
28
|
+
end
|
29
|
+
end
|
30
|
+
assert_equal 'z', Profile.find(2).suffix
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_big_batch
|
34
|
+
Skynet.solo do
|
35
|
+
MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 200, :model_class => "Profile").each do |profile|
|
36
|
+
profile.suffix = 'z'
|
37
|
+
profile.save
|
38
|
+
end
|
39
|
+
end
|
40
|
+
assert_equal 'z', Profile.find(2).suffix
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_joins
|
44
|
+
p = Profile.find(1)
|
45
|
+
p.details.zodiac_sign = 'hermit'
|
46
|
+
p.details.save
|
47
|
+
Skynet.solo do
|
48
|
+
MapreduceLib::AllModels.find(:all, :conditions => "profile_details.zodiac_sign='hermit'", :joins => "JOIN profile_details ON profiles.id = profile_details.profile_id", :batch_size => 2, :model_class => "Profile").each do |profile|
|
49
|
+
profile.suffix = 'z'
|
50
|
+
profile.save
|
51
|
+
end
|
52
|
+
end
|
53
|
+
assert_equal 'z', Profile.find(1).suffix
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_stragglers
|
57
|
+
profiles = Profile.find(:all)
|
58
|
+
Skynet.solo do
|
59
|
+
MapreduceLib::AllModels.find(:all, :batch_size => profiles.size-1, :model_class => "Profile").each do |profile|
|
60
|
+
profile.suffix = 'z'
|
61
|
+
profile.save
|
62
|
+
end
|
63
|
+
end
|
64
|
+
profiles.last.reload
|
65
|
+
assert_equal 'z', profiles.last.suffix
|
66
|
+
end
|
67
|
+
|
68
|
+
def test_small_limit
|
69
|
+
Skynet.solo do
|
70
|
+
MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile", :limit => 3).each do |profile|
|
71
|
+
profile.suffix = 'z'
|
72
|
+
profile.save
|
73
|
+
end
|
74
|
+
end
|
75
|
+
assert_equal 'z', Profile.find(2).suffix
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_each_with_proc_exception
|
79
|
+
Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
|
80
|
+
MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
|
81
|
+
raise "BUSTED" if profile.id == 6
|
82
|
+
profile.suffix = 'z'
|
83
|
+
profile.save
|
84
|
+
end
|
85
|
+
end
|
86
|
+
assert_equal 'z', Profile.find(2).suffix
|
87
|
+
assert_equal nil, Profile.find(6).suffix
|
88
|
+
end
|
89
|
+
|
90
|
+
def test_each_with_class_exception
|
91
|
+
Skynet.solo(:SKYNET_LOG_LEVEL => Logger::FATAL) do
|
92
|
+
MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(MapreduceLib::AllModelsExTest)
|
93
|
+
end
|
94
|
+
assert_equal 'k', Profile.find(2).suffix
|
95
|
+
assert_equal nil, Profile.find(8).suffix
|
96
|
+
end
|
97
|
+
|
98
|
+
def test_each_with_class
|
99
|
+
Skynet.solo do
|
100
|
+
MapreduceLib::AllModels.find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
|
101
|
+
end
|
102
|
+
assert_equal 'k', Profile.find(2).suffix
|
103
|
+
end
|
104
|
+
|
105
|
+
def test_distributed_each
|
106
|
+
Skynet.solo do
|
107
|
+
Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each do |profile|
|
108
|
+
profile.suffix = 'gg'
|
109
|
+
profile.save
|
110
|
+
end
|
111
|
+
assert_equal 'gg', Profile.find(2).suffix
|
112
|
+
Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(self.class)
|
113
|
+
assert_equal 'k', Profile.find(2).suffix
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def test_distributed_each_with_symbol
|
118
|
+
Skynet.solo do
|
119
|
+
Profile.distributed_find(:all, :conditions => "claimed = 1", :batch_size => 2, :model_class => "Profile").each(:mark_modified)
|
120
|
+
end
|
121
|
+
assert ModifiedProfile.find(:first, :conditions => "profile_id = 2")
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.each(profile)
|
125
|
+
profile.suffix = 'k'
|
126
|
+
profile.save
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
130
|
+
|
131
|
+
class AllModelsExTest
|
132
|
+
def self.each(profile)
|
133
|
+
raise "BUSTED" if profile.id == 8
|
134
|
+
profile.suffix = 'k'
|
135
|
+
profile.save
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
@@ -0,0 +1,199 @@
|
|
1
|
+
ENV["RAILS_ENV"] = "test"
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
require '../lib/skynet.rb'
|
5
|
+
|
6
|
+
|
7
|
+
# Tests for the partitioners.
|
8
|
+
#
|
9
|
+
class MysqlMessageQueueTest < Test::Unit::TestCase
|
10
|
+
|
11
|
+
def setup
|
12
|
+
Skynet.configure(
|
13
|
+
:ENABLE => false,
|
14
|
+
# :SKYNET_PIDS_FILE => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
|
15
|
+
:SKYNET_LOG_FILE => STDOUT,
|
16
|
+
:SKYNET_LOG_LEVEL => Logger::ERROR,
|
17
|
+
:MESSAGE_QUEUE_ADAPTER => "Skynet::MessageQueueAdapter::Mysql",
|
18
|
+
:NEXT_TASK_TIMEOUT => 1
|
19
|
+
)
|
20
|
+
mq.clear_outstanding_tasks
|
21
|
+
mq.clear_worker_status
|
22
|
+
|
23
|
+
@worker_message = Skynet::Message.new(
|
24
|
+
:tasktype=> :task,
|
25
|
+
:drburi=> "localhost",
|
26
|
+
:job_id => 1,
|
27
|
+
:task_id => 2,
|
28
|
+
:payload => "payload",
|
29
|
+
:payload_type => "task",
|
30
|
+
:expiry => 20,
|
31
|
+
:expire_time => 1095108406.9251,
|
32
|
+
:iteration => 0,
|
33
|
+
:name => "name",
|
34
|
+
:version => 1
|
35
|
+
)
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
### THE FOLLOWING TESTS THE MessageQueueProxy API
|
40
|
+
|
41
|
+
def test_write_message
|
42
|
+
mq.write_message(@worker_message,10)
|
43
|
+
message = SkynetMessageQueue.find(:all).first
|
44
|
+
assert_equal @worker_message.expiry.to_f, 20
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_template_to_conditions
|
48
|
+
conditions = mq.template_to_conditions(Skynet::Message.next_task_template(1,:task))
|
49
|
+
assert_match "iteration BETWEEN 0 AND 6", conditions
|
50
|
+
assert_match "version = 1", conditions
|
51
|
+
assert_match "tasktype = 'task'", conditions
|
52
|
+
assert_match "expire_time BETWEEN 0 AND", conditions
|
53
|
+
assert_match "payload_type = 'task'", conditions
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_message_to_conditions
|
57
|
+
conditions = mq.template_to_conditions(Skynet::Message.new(Skynet::Message.next_task_template(1)))
|
58
|
+
assert_match "iteration BETWEEN 0 AND 6", conditions
|
59
|
+
assert_match "version = 1", conditions
|
60
|
+
assert_match "tasktype = 'task'", conditions
|
61
|
+
assert_match "expire_time BETWEEN 0 AND", conditions
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_take_next_task
|
65
|
+
assert mq.write_message(@worker_message,10)
|
66
|
+
task = mq.take_next_task(1,1,:task)
|
67
|
+
assert_equal @worker_message.payload, task.payload
|
68
|
+
|
69
|
+
message = SkynetMessageQueue.find(:first)
|
70
|
+
assert_equal 1, message.iteration
|
71
|
+
assert @worker_message.expire_time < message.expire_time
|
72
|
+
excep = nil
|
73
|
+
begin
|
74
|
+
mq.take_next_task(1)
|
75
|
+
rescue Skynet::RequestExpiredError
|
76
|
+
excep = true
|
77
|
+
end
|
78
|
+
assert excep
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_task_failover
|
82
|
+
message = @worker_message.clone
|
83
|
+
message.expiry=0.4
|
84
|
+
assert mq.write_message(message)
|
85
|
+
task = mq.take_next_task(1)
|
86
|
+
assert_equal 2, task.task_id
|
87
|
+
assert_equal 0, task.iteration
|
88
|
+
sleep 0.6
|
89
|
+
ntt = Skynet::Message.next_task_template(1)
|
90
|
+
next_task = mq.take_next_task(1,0.00001)
|
91
|
+
assert_equal 2, next_task.task_id
|
92
|
+
assert_equal 1, next_task.iteration
|
93
|
+
end
|
94
|
+
|
95
|
+
def test_write_and_take_result
|
96
|
+
assert mq.write_message(@worker_message,10)
|
97
|
+
message = mq.take_next_task(1)
|
98
|
+
set_result = {:blah => ['hi']}
|
99
|
+
|
100
|
+
mq.write_result(message,set_result,20)
|
101
|
+
|
102
|
+
assert mq.list_tasks.empty?
|
103
|
+
assert_equal 1, mq.list_results.size
|
104
|
+
result = mq.take_result(1)
|
105
|
+
assert_equal set_result, result.payload
|
106
|
+
end
|
107
|
+
|
108
|
+
def test_write_error
|
109
|
+
mq.write_message(@worker_message,10)
|
110
|
+
message = mq.take_next_task(1)
|
111
|
+
mq.write_error(message,"something_bad_happened",10)
|
112
|
+
result = mq.take_result(1)
|
113
|
+
assert_equal "something_bad_happened", result.payload
|
114
|
+
end
|
115
|
+
|
116
|
+
def test_write_complex_error
|
117
|
+
mq.write_message(@worker_message,10)
|
118
|
+
message = mq.take_next_task(1)
|
119
|
+
error = {:error => "something_bad_happened"}
|
120
|
+
mq.write_error(message,error,10)
|
121
|
+
result = mq.take_result(1)
|
122
|
+
assert_equal error, result.payload
|
123
|
+
end
|
124
|
+
|
125
|
+
def test_write_worker_status
|
126
|
+
assert mq.write_worker_status({
|
127
|
+
:worker_id => 5,
|
128
|
+
:hostname => 'localhost',
|
129
|
+
:process_id => $$,
|
130
|
+
:name => "waiting for master or tasks",
|
131
|
+
:iteration => 0,
|
132
|
+
:processed => 0,
|
133
|
+
:version => 1,
|
134
|
+
:started_at => Time.now.to_i
|
135
|
+
})
|
136
|
+
ws = SkynetWorkerQueue.find_by_worker_id(5)
|
137
|
+
assert_equal ws.iteration, 0
|
138
|
+
|
139
|
+
assert mq.write_worker_status({
|
140
|
+
:worker_id => 5,
|
141
|
+
:hostname => 'localhost',
|
142
|
+
:process_id => $$,
|
143
|
+
:name => "waiting for master or tasks",
|
144
|
+
:processed => 0,
|
145
|
+
:version => 1,
|
146
|
+
:started_at => Time.now.to_i
|
147
|
+
})
|
148
|
+
ws.reload
|
149
|
+
assert_equal ws.iteration, nil
|
150
|
+
end
|
151
|
+
|
152
|
+
def test_take_worker_status
|
153
|
+
assert mq.write_worker_status({
|
154
|
+
:worker_id => 5,
|
155
|
+
:hostname => 'localhost',
|
156
|
+
:process_id => $$,
|
157
|
+
:name => "waiting for master or tasks",
|
158
|
+
:processed => 0,
|
159
|
+
:version => 1,
|
160
|
+
:started_at => Time.now.to_i
|
161
|
+
})
|
162
|
+
|
163
|
+
status = mq.take_worker_status({:hostname => 'localhost',:process_id => $$ })
|
164
|
+
assert_equal $$, status.process_id
|
165
|
+
end
|
166
|
+
|
167
|
+
def test_read_all_worker_statuses
|
168
|
+
assert mq.write_worker_status({
|
169
|
+
:worker_id => 5,
|
170
|
+
:hostname => 'localhost',
|
171
|
+
:process_id => $$,
|
172
|
+
:name => "waiting for master or tasks",
|
173
|
+
:processed => 0,
|
174
|
+
:version => 1,
|
175
|
+
:started_at => Time.now.to_i
|
176
|
+
})
|
177
|
+
status = mq.read_all_worker_statuses.first
|
178
|
+
|
179
|
+
# status = mq.take_worker_status({:hostname => 'localhost',:process_id => $$ })
|
180
|
+
assert_equal $$, status.process_id
|
181
|
+
end
|
182
|
+
|
183
|
+
def test_worker_version
|
184
|
+
mq.set_worker_version(2)
|
185
|
+
assert_equal 2, mq.get_worker_version
|
186
|
+
mq.set_worker_version(10)
|
187
|
+
assert_equal 10, mq.get_worker_version
|
188
|
+
mq.set_worker_version(11)
|
189
|
+
mq.set_worker_version(12)
|
190
|
+
assert_equal 1, SkynetWorkerQueue.count(:id, :conditions => "tasktype = 'workerversion'")
|
191
|
+
end
|
192
|
+
|
193
|
+
private
|
194
|
+
|
195
|
+
def mq
|
196
|
+
Skynet::MessageQueueAdapter::Mysql.new
|
197
|
+
end
|
198
|
+
|
199
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
ENV["RAILS_ENV"] = "test"
|
2
|
+
|
3
|
+
require 'test/unit'
|
4
|
+
require 'pp'
|
5
|
+
|
6
|
+
|
7
|
+
# Tests for the partitioners.
|
8
|
+
#
|
9
|
+
|
10
|
+
|
11
|
+
class SkynetManagerTest < Test::Unit::TestCase
|
12
|
+
|
13
|
+
PIDS = []
|
14
|
+
PIDFILE = []
|
15
|
+
|
16
|
+
def setup
|
17
|
+
Skynet.configure(
|
18
|
+
:ENABLE => false,
|
19
|
+
# :SKYNET_PIDS_FILE => File.expand_path("#{RAILS_ROOT}/log/skynet_#{RAILS_ENV}.pids"),
|
20
|
+
:SKYNET_LOG_FILE => STDOUT,
|
21
|
+
:SKYNET_LOG_LEVEL => Logger::INFO,
|
22
|
+
:MESSAGE_QUEUE_ADAPTER => "Skynet::MessageQueueAdapter::Mysql",
|
23
|
+
:NEXT_TASK_TIMEOUT => 1,
|
24
|
+
:WORKER_CHECK_DELAY => 0.5,
|
25
|
+
:WORKER_WARMUP_TIME => 0.5
|
26
|
+
)
|
27
|
+
mq.clear_outstanding_tasks
|
28
|
+
mq.clear_worker_status
|
29
|
+
@pids = []
|
30
|
+
@pidfile = []
|
31
|
+
|
32
|
+
PIDS.delete_if {true}
|
33
|
+
PIDFILE.delete_if {true}
|
34
|
+
|
35
|
+
Process.stubs(:detatch).returns(true)
|
36
|
+
Process.stubs(:kill).returns(true)
|
37
|
+
|
38
|
+
@minstance = Skynet::Manager.any_instance
|
39
|
+
|
40
|
+
Skynet::Worker.any_instance.stubs(:max_memory_reached?).returns(false)
|
41
|
+
|
42
|
+
@manager = Skynet::Manager.new("path",2)
|
43
|
+
|
44
|
+
def @manager.fork
|
45
|
+
newpid = SkynetManagerTest::PIDS.size + 1
|
46
|
+
SkynetManagerTest::PIDS << newpid
|
47
|
+
worker = Skynet::Worker.new()
|
48
|
+
worker.stubs(:process_id).returns(newpid)
|
49
|
+
worker.notify_worker_queue(:started)
|
50
|
+
newpid
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def test_manager_start
|
55
|
+
@manager.start_workers
|
56
|
+
assert_equal 2, @manager.worker_pids.size
|
57
|
+
assert_equal PIDS.sort, @manager.worker_pids.sort
|
58
|
+
assert_equal PIDS.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
|
59
|
+
end
|
60
|
+
|
61
|
+
def test_check_workers
|
62
|
+
Skynet::Manager.any_instance.expects(:worker_alive?).times(2).returns(true)
|
63
|
+
@manager.start_workers
|
64
|
+
@manager.check_workers
|
65
|
+
assert_equal 2, @manager.worker_pids.size
|
66
|
+
assert_equal PIDS.sort, @manager.worker_pids.sort
|
67
|
+
assert_equal PIDS.sort, @manager.worker_queue.collect {|q|q.process_id}.sort
|
68
|
+
end
|
69
|
+
|
70
|
+
def test_running_pids
|
71
|
+
Skynet::Manager.any_instance.expects(:worker_alive?).with(1).returns(true)
|
72
|
+
Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(false)
|
73
|
+
@manager.start_workers
|
74
|
+
@manager.check_workers
|
75
|
+
assert_equal 1, @manager.worker_pids.size
|
76
|
+
assert_equal [1], @manager.worker_pids.sort
|
77
|
+
assert_equal [1], @manager.worker_queue.collect {|q|q.process_id}.sort
|
78
|
+
end
|
79
|
+
|
80
|
+
## XXX FIXME. What happens if there's a worker missing from the pidfile, but was running?
|
81
|
+
def test_more_in_pidfile_than_queue_alive
|
82
|
+
Skynet::Manager.any_instance.expects(:worker_alive?).with(1).returns(true)
|
83
|
+
Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(true)
|
84
|
+
@manager.start_workers
|
85
|
+
SkynetWorkerQueue.find(:first, :conditions => "process_id = 1").destroy
|
86
|
+
@manager.check_workers
|
87
|
+
assert_equal 1, @manager.worker_pids.size
|
88
|
+
assert_equal [2], @manager.worker_pids.sort
|
89
|
+
assert_equal [2], @manager.worker_queue.collect {|q|q.process_id}.sort
|
90
|
+
end
|
91
|
+
|
92
|
+
def test_dead_workers
|
93
|
+
Skynet::Manager.any_instance.expects(:worker_alive?).times(1).with(1).returns(false)
|
94
|
+
Skynet::Manager.any_instance.expects(:worker_alive?).with(2).returns(true)
|
95
|
+
@manager.start_workers
|
96
|
+
@manager.check_workers
|
97
|
+
assert_equal [2], @manager.worker_pids.sort
|
98
|
+
assert_equal [2], @manager.worker_queue.collect {|q|q.process_id}.sort
|
99
|
+
end
|
100
|
+
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
def mq
|
105
|
+
Skynet::MessageQueueAdapter::Mysql.new
|
106
|
+
end
|
107
|
+
end
|