skynet 0.9.1 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. data/History.txt +99 -0
  2. data/Manifest.txt +10 -9
  3. data/README.txt +74 -7
  4. data/app_generators/skynet_install/skynet_install_generator.rb +26 -22
  5. data/app_generators/skynet_install/templates/migration.rb +11 -5
  6. data/app_generators/skynet_install/templates/skynet +25 -12
  7. data/app_generators/skynet_install/templates/skynet_schema.sql +56 -0
  8. data/bin/skynet +26 -2
  9. data/bin/skynet_install +24 -0
  10. data/bin/skynet_tuplespace_server +13 -0
  11. data/config/hoe.rb +1 -0
  12. data/lib/skynet.rb +3 -0
  13. data/lib/skynet/mapreduce_helper.rb +74 -0
  14. data/lib/skynet/message_queue_adapters/mysql.rb +225 -172
  15. data/lib/skynet/message_queue_adapters/tuple_space.rb +31 -16
  16. data/lib/skynet/skynet_active_record_extensions.rb +78 -46
  17. data/lib/skynet/skynet_config.rb +162 -23
  18. data/lib/skynet/skynet_console.rb +23 -10
  19. data/lib/skynet/skynet_console_helper.rb +61 -58
  20. data/lib/skynet/skynet_job.rb +741 -493
  21. data/lib/skynet/skynet_launcher.rb +5 -1
  22. data/lib/skynet/skynet_manager.rb +106 -49
  23. data/lib/skynet/skynet_message.rb +169 -174
  24. data/lib/skynet/skynet_message_queue.rb +29 -16
  25. data/lib/skynet/skynet_partitioners.rb +92 -0
  26. data/lib/skynet/skynet_ruby_extensions.rb +3 -4
  27. data/lib/skynet/skynet_task.rb +61 -19
  28. data/lib/skynet/skynet_tuplespace_server.rb +0 -2
  29. data/lib/skynet/skynet_worker.rb +73 -51
  30. data/lib/skynet/version.rb +1 -1
  31. data/test/test_active_record_extensions.rb +138 -0
  32. data/test/test_helper.rb +6 -0
  33. data/test/{mysql_message_queue_adaptor_test.rb → test_mysql_message_queue_adapter.rb} +94 -30
  34. data/test/test_skynet.rb +11 -11
  35. data/test/test_skynet_install_generator.rb +0 -4
  36. data/test/test_skynet_job.rb +717 -0
  37. data/test/test_skynet_manager.rb +142 -0
  38. data/test/test_skynet_message.rb +229 -0
  39. data/test/test_skynet_task.rb +24 -0
  40. data/test/{tuplespace_message_queue_test.rb → test_tuplespace_message_queue.rb} +25 -30
  41. data/website/index.html +56 -16
  42. data/website/index.txt +55 -25
  43. data/website/template.rhtml +1 -1
  44. metadata +29 -13
  45. data/app_generators/skynet_install/templates/skynet_console +0 -16
  46. data/bin/skynet_console +0 -9
  47. data/sometest.rb +0 -23
  48. data/test/all_models_test.rb +0 -139
  49. data/test/skynet_manager_test.rb +0 -107
  50. data/test/skynet_message_test.rb +0 -42
  51. data/tmtags +0 -1242
@@ -1,7 +1,11 @@
1
+ # FIXME: should be a module
1
2
  class Skynet
2
3
  include SkynetDebugger
3
4
  def self.new(options={})
4
- if options[:worker_type] or ARGV.detect {|a| a =~ /worker_type/ }
5
+ if ARGV.detect {|a| a == 'console' }
6
+ ARGV.delete('console')
7
+ Skynet::Console.start
8
+ elsif options[:worker_type] or ARGV.detect {|a| a =~ /worker_type/ }
5
9
  Skynet::Worker.start(options)
6
10
  else
7
11
  Skynet::Manager.start(options)
@@ -7,6 +7,10 @@ class Skynet
7
7
  end
8
8
 
9
9
  class Manager
10
+
11
+ class Error < StandardError
12
+ end
13
+
10
14
  include SkynetDebugger
11
15
 
12
16
  Skynet::CONFIG[:PERCENTAGE_OF_TASK_ONLY_WORKERS] ||= 0.7
@@ -16,27 +20,31 @@ class Skynet
16
20
  "MANAGER"
17
21
  end
18
22
 
19
-
20
- attr_accessor :required_libs
23
+ attr_accessor :required_libs, :queue_id
24
+ attr_reader :config
21
25
 
22
- def initialize(script_path,workers_requested,adlibs=[])
23
- info "Skynet Launcher Path: [#{@script_path}]"
24
- @script_path = script_path
25
- @mutex = Mutex.new
26
- @workers_requested = workers_requested
27
- @required_libs = adlibs
26
+ def initialize(options)
27
+ raise Error.new("You must provide a script path to Skynet::Manager.new.") unless options[:script_path]
28
+ @script_path = options[:script_path]
29
+ # info "Skynet Launcher Path: [#{@script_path}]"
30
+ @workers_requested = options[:workers] || 4
31
+ @required_libs = options[:adlibs] || []
32
+ @queue_id = options[:queue_id] || 0
28
33
  @number_of_workers = 0
29
34
  @workers_by_type = {:master => [], :task => [], :any => []}
30
35
  @signaled_workers = []
31
36
  @workers_running = {}
37
+ @workers_restarting = 0
32
38
  @all_workers_started = false
39
+ @config = Skynet::Config.new
40
+ @mutex = Mutex.new
33
41
  end
34
42
 
35
43
  def start_workers
36
44
  setup_signals
37
45
 
38
46
  starting = workers_to_start(@workers_requested)
39
- warn "Starting #{starting} workers. #{@workers_requested - starting} already running."
47
+ warn "Starting #{starting} workers. QUEUE: #{config.queue_name_by_id(queue_id)} #{@workers_requested - starting} already running."
40
48
  add_worker(starting)
41
49
  end
42
50
 
@@ -61,7 +69,7 @@ class Skynet
61
69
  def check_started_workers
62
70
  workers = []
63
71
  begin
64
- 50.times do |ii|
72
+ 100.times do |ii|
65
73
  workers = worker_queue
66
74
  warn "Checking started workers, #{workers.size} out of #{@number_of_workers} after the #{(ii+1)}th try..."
67
75
  break if workers.size >= @number_of_workers
@@ -97,20 +105,23 @@ class Skynet
97
105
  end
98
106
 
99
107
  def check_workers
100
- q_pids = worker_queue_pids || []
108
+ worker_queue = self.worker_queue
109
+ q_pids = worker_queue_pids(worker_queue) || []
101
110
  info "Checking on #{@number_of_workers} workers..." unless @shutdown
102
- check_running_pids(q_pids)
103
- check_number_of_workers(q_pids)
111
+ check_running_pids(worker_queue)
112
+ check_number_of_workers(worker_queue)
104
113
  true
105
114
  end
106
115
 
107
- def check_running_pids(q_pids)
116
+ def check_running_pids(worker_queue)
117
+ # There are workers running that are not in the queue. When does this happen?
118
+ q_pids = worker_queue_pids(worker_queue) || []
108
119
  if @workers_running.keys.size > q_pids.size
109
120
  (@workers_running.keys - q_pids).each do |wpid|
110
121
  error "Missing worker #{wpid} from worker queue. Removing and/or killing."
111
122
  Process.kill("TERM",wpid) if worker_alive?(wpid)
112
123
  @workers_running.delete(wpid)
113
- q_pids.delete(wpid)
124
+ q_pids.delete(wpid)
114
125
  end
115
126
  end
116
127
 
@@ -125,13 +136,45 @@ class Skynet
125
136
  end
126
137
  q_pids
127
138
  end
139
+
140
+ def check_number_of_workers(worker_queue=self.worker_queue)
141
+ q_pids = worker_queue_pids(worker_queue) || []
142
+ if @shutdown
143
+ worker_shutdown(worker_queue)
144
+ if q_pids.size < 1
145
+ exit
146
+ end
147
+ elsif @workers_restarting > 0
148
+ if @workers_requested - q_pids.size != 0
149
+ restarting = @workers_requested - q_pids.size
150
+ warn "RESTART MODE: Expected #{@number_of_workers} workers. #{q_pids.size} running. #{restarting} are still restarting"
151
+ else
152
+ warn "RESTART MODE: Expected #{@number_of_workers} workers. #{q_pids.size} running."
153
+ end
154
+ @workers_restarting = @workers_requested - q_pids.size
155
+
156
+ elsif q_pids.size != @number_of_workers
157
+ starting = 0
158
+ if q_pids.size.to_f / @workers_requested.to_f < 0.85
159
+ starting = @workers_requested - q_pids.size
160
+ error "Expected #{@number_of_workers} workers. #{q_pids.size} running. Starting #{starting}"
161
+ @number_of_workers = q_pids.size
162
+ add_worker(starting)
163
+ else
164
+
165
+ error "Expected #{@number_of_workers} workers. #{q_pids.size} running."
166
+ @number_of_workers = q_pids.size
167
+ end
168
+ end
169
+ end
128
170
 
129
- def worker_shutdown(q_pids)
171
+ def worker_shutdown(worker_queue)
172
+ q_pids = worker_queue_pids(worker_queue) || []
130
173
  if not @masters_dead
131
- warn "Shutting down masters. #{q_pids.size} workers still running." if q_pids.size > 0
132
174
  workers_to_kill = worker_queue.select do |w|
133
175
  w.map_or_reduce == "master" and @workers_running.include?(w.process_id)
134
176
  end
177
+ warn "Shutting down masters. #{q_pids.size} workers still running." if q_pids.size > 0
135
178
 
136
179
  worker_pids_to_kill = workers_to_kill.collect { |w| w.process_id }
137
180
  if worker_pids_to_kill and not worker_pids_to_kill.empty?
@@ -143,10 +186,10 @@ class Skynet
143
186
  signal_workers("INT")
144
187
  @masters_dead = true
145
188
  sleep 1
146
- return check_number_of_workers(worker_queue_pids)
189
+ return check_number_of_workers()
147
190
  else
148
191
  sleep 4
149
- return check_number_of_workers(worker_queue_pids)
192
+ return check_number_of_workers()
150
193
  end
151
194
  else
152
195
  warn "Shutting down. #{q_pids.size} workers still running." if q_pids.size > 0
@@ -155,27 +198,7 @@ class Skynet
155
198
  info "No more workers running."
156
199
  end
157
200
  end
158
-
159
- def check_number_of_workers(q_pids)
160
- if @shutdown
161
- worker_shutdown(q_pids)
162
- if q_pids.size < 1
163
- exit
164
- end
165
- elsif q_pids.size != @number_of_workers
166
- if q_pids.size.to_f / @workers_requested.to_f < 0.85
167
- starting = @workers_requested - q_pids.size
168
- error "Expected #{@number_of_workers} workers. #{q_pids.size} running. Starting #{starting}"
169
- @number_of_workers += starting
170
- add_worker(starting)
171
- else
172
- error "Expected #{@number_of_workers} workers. #{q_pids.size} running."
173
- @number_of_workers = q_pids.size
174
- end
175
- end
176
-
177
- end
178
-
201
+
179
202
  def take_worker_status(worker_process_id)
180
203
  begin
181
204
  mq.take_worker_status({
@@ -222,11 +245,12 @@ class Skynet
222
245
  worker_types[:any] += 1
223
246
  end
224
247
  cmd = "#{@script_path} --worker_type=#{worker_type}"
248
+ cmd << " --queue_id=#{queue_id}"
225
249
  cmd << " -r #{required_libs.join(' -r ')}" if required_libs and not required_libs.empty?
226
250
  wpid = self.fork_and_exec(cmd)
227
251
  @workers_by_type[worker_type] ||= []
228
252
  @workers_by_type[worker_type] << wpid
229
- warn "Adding Worker ##{ii} PID: #{wpid} WORKER_TYPE?:#{worker_type}"
253
+ warn "Adding Worker ##{ii} PID: #{wpid} QUEUE: #{queue_id}, WORKER_TYPE?:#{worker_type}"
230
254
  @mutex.synchronize do
231
255
  @number_of_workers += 1
232
256
  end
@@ -252,7 +276,6 @@ class Skynet
252
276
  @number_of_workers -= 1
253
277
  @workers_running.delete(wpid)
254
278
  warn "REMOVING WORKER #{wpid}"
255
- # error "SHUTTING DOWN #{wpid} MR:",worker_queue.detect{|w|w.process_id == wpid}
256
279
  @signaled_workers << wpid
257
280
  Process.kill("INT",wpid)
258
281
  end
@@ -290,6 +313,19 @@ class Skynet
290
313
  check_started_workers
291
314
  end
292
315
 
316
+ # ===========================
317
+ # = XXX THIS IS A HORRIBLE HACK =
318
+ # ===========================
319
+ def restart_worker(wpid)
320
+ info "RESTARTING WORKER #{wpid}"
321
+ @mutex.synchronize do
322
+ @workers_running.delete(wpid)
323
+ @workers_restarting += 1
324
+ end
325
+ Process.kill("HUP",wpid)
326
+ sleep Skynet::CONFIG[:WORKER_CHECK_DELAY]
327
+ end
328
+
293
329
  def restart_workers
294
330
  @all_workers_started = false
295
331
  signal_workers("HUP")
@@ -350,7 +386,7 @@ class Skynet
350
386
  mq.read_all_worker_statuses(hostname)
351
387
  end
352
388
 
353
- def worker_queue_pids
389
+ def worker_queue_pids(worker_queue=self.worker_queue)
354
390
  worker_queue.collect {|w| w.process_id}
355
391
  end
356
392
 
@@ -375,8 +411,19 @@ class Skynet
375
411
  options[:remove_workers] ||= nil
376
412
  options[:use_rails] ||= false
377
413
  options[:required_libs] ||= []
414
+ options[:workers] ||= Skynet::CONFIG[:NUMBER_OF_WORKERS] || 4
415
+ options[:pid_file] ||= Skynet::CONFIG[:SKYNET_PIDS_FILE]
416
+ options[:script_path] ||= Skynet::CONFIG[:LAUNCHER_PATH]
417
+
418
+ config = Skynet::Config.new
419
+
378
420
  OptionParser.new do |opt|
379
- opt.banner = "Usage: skynet [options]"
421
+ opt.banner = %{Usage:
422
+ > skynet [options]
423
+
424
+ You can also run:
425
+ > skynet console [options]
426
+ }
380
427
  opt.on('', '--restart-all-workers', 'Restart All Workers') do |v|
381
428
  puts "Restarting ALL workers on ALL machines."
382
429
  begin
@@ -416,13 +463,23 @@ class Skynet
416
463
  opt.on('-r', '--required LIBRARY', 'Require the specified libraries') do |v|
417
464
  options[:required_libs] << File.expand_path(v)
418
465
  end
419
-
466
+ opt.on('-q', '--queue QUEUE_NAME', 'Which queue should these workers use (default "default").') do |v|
467
+ options[:queue] = v
468
+ end
469
+ opt.on('-i', '--queue_id queue_id', 'Which queue should these workers use (default 0).') do |v|
470
+ options[:queue_id] = v.to_i
471
+ end
420
472
  opt.parse!(ARGV)
421
473
  end
474
+ if options[:queue]
475
+ if options[:queue_id]
476
+ raise Skynet::Error.new("You may either provide a queue_id or a queue, but not both.")
477
+ end
478
+ options[:queue_id] = config.queue_id_by_name(options[:queue])
479
+ else
480
+ options[:queue_id] ||= 0
481
+ end
422
482
 
423
- options[:workers] ||= Skynet::CONFIG[:NUMBER_OF_WORKERS] || 4
424
- options[:pid_file] ||= File.dirname(Skynet::CONFIG[:SKYNET_PIDS_FILE]) + "/skynet_worker.pid"
425
-
426
483
  options[:required_libs].each do |adlib|
427
484
  begin
428
485
  require adlib
@@ -469,7 +526,7 @@ class Skynet
469
526
 
470
527
  begin
471
528
  info "STARTING THE MANAGER!!!!!!!!!!!"
472
- @manager = Skynet::Manager.new(Skynet::CONFIG[:LAUNCHER_PATH],options[:workers],options[:required_libs])
529
+ @manager = Skynet::Manager.new(options)
473
530
  DRb.start_service(Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL], @manager)
474
531
  info "WORKER MANAGER URI: #{DRb.uri}"
475
532
  @manager.start_workers
@@ -3,43 +3,59 @@ class Skynet
3
3
 
4
4
  include SkynetDebugger
5
5
 
6
- class BadMessage < Skynet::Error
7
- end
6
+ class BadMessage < Skynet::Error; end
8
7
 
9
8
  class << self
10
9
  attr_accessor :fields
11
10
  end
12
11
 
13
- self.fields = {
14
- 0 => :tasktype,
15
- 1 => :drburi,
16
- 2 => :task_id,
17
- 3 => :job_id,
18
- 4 => :payload,
19
- 5 => :payload_type,
20
- 6 => :name,
21
- 7 => :expiry,
22
- 8 => :expire_time,
23
- 9 => :iteration,
24
- 10 => :version
25
- }
12
+ self.fields = [
13
+ :tasktype,
14
+ :drburi,
15
+ :task_id,
16
+ :job_id,
17
+ :payload,
18
+ :payload_type,
19
+ :name,
20
+ :expiry,
21
+ :expire_time,
22
+ :iteration,
23
+ :version,
24
+ :retry,
25
+ :queue_id
26
+ ]
26
27
 
27
- self.fields.values.each do |method|
28
- next if method == :payload
29
- next if method == :tasktype
30
- next if method == :payload_type
28
+ self.fields.each do |method|
29
+ next if [:payload, :tasktype, :payload_type].include?(method)
31
30
  attr_accessor method
32
31
  end
33
32
 
34
33
  attr_reader :payload_type, :tasktype
34
+
35
+ def self.new_task_message(task,job)
36
+ self.new(
37
+ :job_id => job.job_id,
38
+ :expire_time => job.start_after,
39
+ :version => job.version,
40
+ :queue_id => job.queue_id || 0,
41
+ :iteration => 0,
42
+ :tasktype => :task,
43
+ :task_id => task.task_id,
44
+ :payload => task,
45
+ :payload_type => task.task_or_master,
46
+ :expiry => task.result_timeout,
47
+ :name => task.name,
48
+ :retry => task.retry
49
+ )
50
+ end
35
51
 
36
52
  def initialize(opts)
37
53
  if opts.is_a?(Array)
38
- self.class.fields.each do |ii, field|
54
+ self.class.fields.each_with_index do |field, ii|
39
55
  self.send("#{field}=",opts[ii] || nil)
40
56
  end
41
57
  elsif opts
42
- self.class.fields.values.each do |field|
58
+ self.class.fields.each do |field|
43
59
  value = opts[field] || opts[field.to_s] || nil
44
60
  self.send("#{field}=",value) if value
45
61
  end
@@ -47,6 +63,7 @@ class Skynet
47
63
  if opts_raw_payload
48
64
  self.raw_payload = opts_raw_payload
49
65
  end
66
+ self.retry ||= 0
50
67
  end
51
68
  self.payload
52
69
  end
@@ -95,15 +112,15 @@ class Skynet
95
112
  end
96
113
 
97
114
  def to_a
98
- self.class.fields.keys.sort.collect do |ii|
99
- self.send(self.class.fields[ii])
115
+ self.class.fields.collect do |field|
116
+ self.send(field)
100
117
  end
101
118
  end
102
119
 
103
120
  def to_hash
104
121
  hash = {}
105
- self.class.fields.keys.sort.collect do |ii|
106
- hash[self.class.fields[ii]] = self.send(self.class.fields[ii])
122
+ self.class.fields.each do |field|
123
+ hash[field] = self.send(field)
107
124
  end
108
125
  hash
109
126
  end
@@ -120,85 +137,66 @@ class Skynet
120
137
  expire_time * 2
121
138
  end
122
139
 
123
- ####### TEMPLATES ############
124
- def self.next_task_template(version=nil,payload_type=nil)
125
- fields.keys.sort.collect do |ii|
126
- field = fields[ii]
127
- case field
128
- when :expire_time
129
- (0 .. Time.now.to_i)
130
- when :tasktype
131
- :task
132
- when :version
133
- version
134
- when :payload_type
135
- payload_type
136
- when :iteration
137
- (0..6)
138
- else
139
- nil
140
- end
140
+ def self.next_task_template(version=nil, payload_type=nil, queue_id=0)
141
+ template = {
142
+ :expire_time => (0 .. Time.now.to_i),
143
+ :tasktype => :task,
144
+ :queue_id => queue_id,
145
+ :version => version,
146
+ :payload_type => payload_type,
147
+ :iteration => (0..Skynet::CONFIG[:MAX_RETRIES]),
148
+ }
149
+
150
+ fields.collect do |field|
151
+ template[field]
141
152
  end
142
153
  end
143
154
 
144
155
  def self.result_template(job_id,tasktype=:result)
145
- fields.keys.sort.collect do |ii|
146
- field = fields[ii]
147
- case field
148
- when :tasktype
149
- tasktype
150
- when :job_id
151
- job_id
152
- else
153
- nil
154
- end
155
- end
156
+ template = {
157
+ :tasktype => tasktype,
158
+ :job_id => job_id
159
+ }
160
+ fields.collect do |field|
161
+ template[field]
162
+ end
156
163
  end
157
164
 
158
165
  def self.result_message(message,result,tasktype=:result, resulttype=:result)
159
- message_array = fields.keys.sort.collect do |ii|
160
- field = fields[ii]
161
- case field
162
- when :tasktype
163
- tasktype
164
- when :payload
165
- result
166
- when :payload_type
167
- resulttype
168
- else
169
- message.send(fields[ii])
170
- end
166
+ template = {
167
+ :tasktype => tasktype,
168
+ :payload => result,
169
+ :payload_type => resulttype
170
+ }
171
+
172
+ fields.each do |field|
173
+ template[field] = message.send(field) unless template.has_key?(field)
171
174
  end
172
- new(message_array)
175
+ new(template)
173
176
  end
174
177
 
175
178
  def result_message(result,tasktype=:result, resulttype=:result)
176
179
  self.class.result_message(self,result,tasktype,resulttype)
177
180
  end
178
181
 
179
- def self.outstanding_tasks_template(iteration=nil)
180
- fields.keys.sort.collect do |ii|
181
- field = fields[ii]
182
- case field
183
- when :tasktype
184
- :task
185
- when :iteration
186
- iteration
187
- else
188
- nil
189
- end
182
+ def self.outstanding_tasks_template(iteration=nil,queue_id=0)
183
+ template = {
184
+ :tasktype => :task,
185
+ :queue_id => queue_id,
186
+ :iteration => iteration
187
+ }
188
+ fields.collect do |field|
189
+ template[field]
190
190
  end
191
191
  end
192
192
 
193
- def self.outstanding_results_template
194
- fields.keys.sort.collect do |ii|
195
- field = fields[ii]
196
- case field
197
- when :tasktype
198
- :result
199
- else
200
- nil
201
- end
193
+ def self.outstanding_results_template(queue_id=0)
194
+ template = {
195
+ :tasktype => :result,
196
+ :queue_id => queue_id
197
+ }
198
+ fields.collect do |field|
199
+ template[field]
202
200
  end
203
201
  end
204
202
 
@@ -211,16 +209,15 @@ class Skynet
211
209
  end
212
210
 
213
211
  def self.error_template(message)
214
- fields.keys.sort.collect do |ii|
215
- field = fields[ii]
216
- case field
217
- when :tasktype
218
- message.tasktype
219
- when :drburi, :version, :task_id
220
- message.send(fields[ii])
221
- else
222
- nil
223
- end
212
+ template = {
213
+ :tasktype => message.tasktype,
214
+ :drburi => message.drburi,
215
+ :version => message.version,
216
+ :task_id => message.task_id,
217
+ :queue_id => message.queue_id
218
+ }
219
+ fields.collect do |field|
220
+ template[field]
224
221
  end
225
222
  end
226
223
 
@@ -229,19 +226,29 @@ class Skynet
229
226
  end
230
227
 
231
228
  def self.fallback_task_message(message)
232
- opts = Hash.new
233
- fields.values.each do |field|
234
- case field
235
- when :iteration
236
- opts[:iteration] = message.iteration + 1
237
- when :expire_time
238
- opts[:expire_time] = Time.now.to_i + message.expiry
229
+ template = {}
230
+ if message.retry
231
+ if (message.retry and message.iteration >= message.retry)
232
+ template[:iteration] = -1
239
233
  else
240
- opts[field] = message.send(field)
234
+ template[:iteration] = message.iteration + 1
241
235
  end
236
+ # Originally I was gonna do this for map and reduce, but we don't know that here, just whether its a master.
237
+ elsif message.payload_type.to_sym == :master and Skynet::CONFIG[:DEFAULT_MASTER_RETRY] and message.iteration >= Skynet::CONFIG[:DEFAULT_MASTER_RETRY]
238
+ template[:iteration] = -1
239
+ elsif Skynet::CONFIG[:MAX_RETRIES] and message.iteration >= Skynet::CONFIG[:MAX_RETRIES]
240
+ template[:iteration] = -1
241
+ else
242
+ template[:iteration] = message.iteration + 1
242
243
  end
243
- # debug "BUILDING NEXT FALLBACK TASK MESSAGE OFF"#, opts
244
- Skynet::Message.new(opts)
244
+
245
+ template[:expire_time] = Time.now.to_i + message.expiry
246
+
247
+ fields.each do |field|
248
+ template[field] = message.send(field) unless template.has_key?(field)
249
+ end
250
+ # debug "BUILDING NEXT FALLBACK TASK MESSAGE OFF"#, template
251
+ Skynet::Message.new(template)
245
252
  end
246
253
 
247
254
  def fallback_task_message
@@ -249,18 +256,16 @@ class Skynet
249
256
  end
250
257
 
251
258
  def self.fallback_template(message)
252
- fields.keys.sort.collect do |ii|
253
- field = fields[ii]
254
- case field
255
- when :tasktype
256
- message.tasktype
257
- when :drburi, :version, :task_id
258
- message.send(field)
259
- when :iteration
260
- (1..20)
261
- else
262
- nil
263
- end
259
+ template = {
260
+ :tasktype => message.tasktype,
261
+ :drburi => message.drburi,
262
+ :version => message.version,
263
+ :task_id => message.task_id,
264
+ :queue_id => message.queue_id,
265
+ :iteration => (1..Skynet::CONFIG[:MAX_RETRIES]),
266
+ }
267
+ fields.collect do |field|
268
+ template[field]
264
269
  end
265
270
  end
266
271
 
@@ -285,49 +290,43 @@ class Skynet
285
290
  end
286
291
 
287
292
  def self.template
288
- fields.keys.sort.collect do |ii|
289
- field = fields[ii]
290
- case field
291
- when :tasktype
292
- :current_worker_rev
293
- else
294
- nil
295
- end
293
+ template = {
294
+ :tasktype => :current_worker_rev
295
+ }
296
+ fields.collect do |field|
297
+ template[field]
296
298
  end
297
299
  end
298
300
 
299
301
  def template
300
- fields.keys.sort.collect do |ii|
301
- field = fields[ii]
302
- case field
303
- when :tasktype
304
- :current_worker_rev
305
- when :expire_time
306
- nil
307
- else
308
- self.send(field)
309
- end
302
+ template = {
303
+ :tasktype => :current_worker_rev,
304
+ :expire_time => nil
305
+ }
306
+ fields.collect do |field|
307
+ template[field] || self.send(field)
310
308
  end
311
309
  end
312
310
  end
313
311
 
314
312
  class WorkerStatusMessage < Skynet::Message
315
- self.fields = {
316
- 0 => :tasktype,
317
- 1 => :tasksubtype,
318
- 2 => :worker_id,
319
- 3 => :hostname,
320
- 4 => :process_id,
321
- 5 => :job_id,
322
- 6 => :task_id,
323
- 7 => :iteration,
324
- 8 => :name,
325
- 9 => :map_or_reduce,
326
- 10 => :started_at,
327
- 11 => :version,
328
- 12 => :processed
329
- }
330
- self.fields.values.each { |method| attr_accessor method }
313
+ self.fields = [
314
+ :tasktype,
315
+ :tasksubtype,
316
+ :worker_id,
317
+ :hostname,
318
+ :process_id,
319
+ :job_id,
320
+ :task_id,
321
+ :iteration,
322
+ :name,
323
+ :map_or_reduce,
324
+ :started_at,
325
+ :version,
326
+ :processed,
327
+ :queue_id
328
+ ]
329
+ self.fields.each { |method| attr_accessor method }
331
330
 
332
331
  def initialize(opts)
333
332
  super
@@ -336,31 +335,27 @@ class Skynet
336
335
  end
337
336
 
338
337
  def self.worker_status_template(opts)
339
- fields.keys.sort.collect do |key|
340
- case fields[key]
341
- when :tasktype : :status
342
- when :tasksubtype : :worker
343
- when :hostname : opts[:hostname]
344
- when :process_id : opts[:process_id]
345
- else
346
- nil
347
- end
338
+ template = {
339
+ :tasktype => :status,
340
+ :tasksubtype => :worker,
341
+ :hostname => opts[:hostname],
342
+ :process_id => opts[:process_id]
343
+ }
344
+ fields.collect do |field|
345
+ template[field]
348
346
  end
349
347
  end
350
348
 
351
349
  def self.all_workers_template(hostname=nil)
352
- fields.keys.sort.collect do |key|
353
- case fields[key]
354
- when :tasktype : :status
355
- when :tasksubtype : :worker
356
- when :hostname
357
- hostname if hostname
358
- else
359
- nil
360
- end
350
+ template = {
351
+ :tasktype => :status,
352
+ :tasksubtype => :worker,
353
+ :hostname => hostname,
354
+ }
355
+ fields.collect do |field|
356
+ template[field]
361
357
  end
362
- end
363
-
364
- end
358
+ end
359
+ end # class WorkerStatusMessage
365
360
 
366
361
  end