skynet 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +4 -0
- data/License.txt +20 -0
- data/Manifest.txt +65 -0
- data/README.txt +100 -0
- data/Rakefile +4 -0
- data/app_generators/skynet_install/USAGE +5 -0
- data/app_generators/skynet_install/skynet_install_generator.rb +84 -0
- data/app_generators/skynet_install/templates/migration.rb +60 -0
- data/app_generators/skynet_install/templates/skynet +33 -0
- data/app_generators/skynet_install/templates/skynet_console +16 -0
- data/bin/skynet +20 -0
- data/bin/skynet_console +9 -0
- data/bin/skynet_install +12 -0
- data/bin/skynet_tuplespace_server +53 -0
- data/config/hoe.rb +74 -0
- data/config/requirements.rb +17 -0
- data/lib/skynet.rb +34 -0
- data/lib/skynet/mapreduce_test.rb +25 -0
- data/lib/skynet/message_queue_adapters/message_queue_adapter.rb +70 -0
- data/lib/skynet/message_queue_adapters/mysql.rb +573 -0
- data/lib/skynet/message_queue_adapters/tuple_space.rb +327 -0
- data/lib/skynet/skynet_active_record_extensions.rb +237 -0
- data/lib/skynet/skynet_config.rb +59 -0
- data/lib/skynet/skynet_console.rb +34 -0
- data/lib/skynet/skynet_console_helper.rb +59 -0
- data/lib/skynet/skynet_debugger.rb +84 -0
- data/lib/skynet/skynet_guid_generator.rb +68 -0
- data/lib/skynet/skynet_job.rb +607 -0
- data/lib/skynet/skynet_launcher.rb +10 -0
- data/lib/skynet/skynet_logger.rb +52 -0
- data/lib/skynet/skynet_manager.rb +486 -0
- data/lib/skynet/skynet_message.rb +366 -0
- data/lib/skynet/skynet_message_queue.rb +100 -0
- data/lib/skynet/skynet_ruby_extensions.rb +36 -0
- data/lib/skynet/skynet_task.rb +76 -0
- data/lib/skynet/skynet_tuplespace_server.rb +82 -0
- data/lib/skynet/skynet_worker.rb +395 -0
- data/lib/skynet/version.rb +9 -0
- data/log/debug.log +0 -0
- data/log/skynet.log +29 -0
- data/log/skynet_tuplespace_server.log +7 -0
- data/log/skynet_worker.pid +1 -0
- data/script/destroy +14 -0
- data/script/generate +14 -0
- data/script/txt2html +74 -0
- data/setup.rb +1585 -0
- data/sometest.rb +23 -0
- data/tasks/deployment.rake +34 -0
- data/tasks/environment.rake +7 -0
- data/tasks/website.rake +17 -0
- data/test/all_models_test.rb +139 -0
- data/test/mysql_message_queue_adaptor_test.rb +199 -0
- data/test/skynet_manager_test.rb +107 -0
- data/test/skynet_message_test.rb +42 -0
- data/test/test_generator_helper.rb +20 -0
- data/test/test_helper.rb +2 -0
- data/test/test_skynet.rb +11 -0
- data/test/test_skynet_install_generator.rb +53 -0
- data/test/tuplespace_message_queue_test.rb +179 -0
- data/tmtags +1242 -0
- data/website/index.html +93 -0
- data/website/index.txt +39 -0
- data/website/javascripts/rounded_corners_lite.inc.js +285 -0
- data/website/stylesheets/screen.css +138 -0
- data/website/template.rhtml +48 -0
- metadata +129 -0
@@ -0,0 +1,366 @@
|
|
1
|
+
class Skynet
|
2
|
+
class Message
|
3
|
+
|
4
|
+
include SkynetDebugger
|
5
|
+
|
6
|
+
class BadMessage < Skynet::Error
|
7
|
+
end
|
8
|
+
|
9
|
+
class << self
|
10
|
+
attr_accessor :fields
|
11
|
+
end
|
12
|
+
|
13
|
+
self.fields = {
|
14
|
+
0 => :tasktype,
|
15
|
+
1 => :drburi,
|
16
|
+
2 => :task_id,
|
17
|
+
3 => :job_id,
|
18
|
+
4 => :payload,
|
19
|
+
5 => :payload_type,
|
20
|
+
6 => :name,
|
21
|
+
7 => :expiry,
|
22
|
+
8 => :expire_time,
|
23
|
+
9 => :iteration,
|
24
|
+
10 => :version
|
25
|
+
}
|
26
|
+
|
27
|
+
self.fields.values.each do |method|
|
28
|
+
next if method == :payload
|
29
|
+
next if method == :tasktype
|
30
|
+
next if method == :payload_type
|
31
|
+
attr_accessor method
|
32
|
+
end
|
33
|
+
|
34
|
+
attr_reader :payload_type, :tasktype
|
35
|
+
|
36
|
+
def initialize(opts)
|
37
|
+
if opts.is_a?(Array)
|
38
|
+
self.class.fields.each do |ii, field|
|
39
|
+
self.send("#{field}=",opts[ii] || nil)
|
40
|
+
end
|
41
|
+
elsif opts
|
42
|
+
self.class.fields.values.each do |field|
|
43
|
+
value = opts[field] || opts[field.to_s] || nil
|
44
|
+
self.send("#{field}=",value) if value
|
45
|
+
end
|
46
|
+
opts_raw_payload = opts[:raw_payload] || opts["raw_payload"]
|
47
|
+
if opts_raw_payload
|
48
|
+
self.raw_payload = opts_raw_payload
|
49
|
+
end
|
50
|
+
end
|
51
|
+
self.payload
|
52
|
+
end
|
53
|
+
|
54
|
+
def fields
|
55
|
+
self.class.fields
|
56
|
+
end
|
57
|
+
|
58
|
+
def tasktype=(ttype)
|
59
|
+
@tasktype = ttype.to_sym
|
60
|
+
end
|
61
|
+
|
62
|
+
def payload_type=(ptype)
|
63
|
+
@payload_type = ptype.to_sym if ptype
|
64
|
+
end
|
65
|
+
|
66
|
+
# alias for payload
|
67
|
+
def task
|
68
|
+
payload
|
69
|
+
end
|
70
|
+
|
71
|
+
def payload=(data)
|
72
|
+
@payload = data
|
73
|
+
self.raw_payload = data.to_yaml if data.respond_to?(:to_yaml) and not payload.kind_of?(Proc)
|
74
|
+
end
|
75
|
+
|
76
|
+
def payload
|
77
|
+
@payload ||= begin
|
78
|
+
YAML::load(self.raw_payload) if self.raw_payload
|
79
|
+
rescue Exception => e
|
80
|
+
raise BadMessage.new("Couldnt marshal payload #{e.inspect} #{e.backtrace.join("\n")}")
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def raw_payload=(data)
|
85
|
+
@raw_payload = data
|
86
|
+
@payload=nil
|
87
|
+
end
|
88
|
+
|
89
|
+
def raw_payload
|
90
|
+
@raw_payload
|
91
|
+
end
|
92
|
+
|
93
|
+
def [](ii)
|
94
|
+
send(self.class.fields[ii])
|
95
|
+
end
|
96
|
+
|
97
|
+
def to_a
|
98
|
+
self.class.fields.keys.sort.collect do |ii|
|
99
|
+
self.send(self.class.fields[ii])
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def to_hash
|
104
|
+
hash = {}
|
105
|
+
self.class.fields.keys.sort.collect do |ii|
|
106
|
+
hash[self.class.fields[ii]] = self.send(self.class.fields[ii])
|
107
|
+
end
|
108
|
+
hash
|
109
|
+
end
|
110
|
+
|
111
|
+
def to_h
|
112
|
+
to_hash
|
113
|
+
end
|
114
|
+
|
115
|
+
def to_s
|
116
|
+
to_a
|
117
|
+
end
|
118
|
+
|
119
|
+
def timeout
|
120
|
+
expire_time * 2
|
121
|
+
end
|
122
|
+
|
123
|
+
####### TEMPLATES ############
|
124
|
+
def self.next_task_template(version=nil,payload_type=nil)
|
125
|
+
fields.keys.sort.collect do |ii|
|
126
|
+
field = fields[ii]
|
127
|
+
case field
|
128
|
+
when :expire_time
|
129
|
+
(0 .. Time.now.to_i)
|
130
|
+
when :tasktype
|
131
|
+
:task
|
132
|
+
when :version
|
133
|
+
version
|
134
|
+
when :payload_type
|
135
|
+
payload_type
|
136
|
+
when :iteration
|
137
|
+
(0..6)
|
138
|
+
else
|
139
|
+
nil
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def self.result_template(job_id,tasktype=:result)
|
145
|
+
fields.keys.sort.collect do |ii|
|
146
|
+
field = fields[ii]
|
147
|
+
case field
|
148
|
+
when :tasktype
|
149
|
+
tasktype
|
150
|
+
when :job_id
|
151
|
+
job_id
|
152
|
+
else
|
153
|
+
nil
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def self.result_message(message,result,tasktype=:result, resulttype=:result)
|
159
|
+
message_array = fields.keys.sort.collect do |ii|
|
160
|
+
field = fields[ii]
|
161
|
+
case field
|
162
|
+
when :tasktype
|
163
|
+
tasktype
|
164
|
+
when :payload
|
165
|
+
result
|
166
|
+
when :payload_type
|
167
|
+
resulttype
|
168
|
+
else
|
169
|
+
message.send(fields[ii])
|
170
|
+
end
|
171
|
+
end
|
172
|
+
new(message_array)
|
173
|
+
end
|
174
|
+
|
175
|
+
def result_message(result,tasktype=:result, resulttype=:result)
|
176
|
+
self.class.result_message(self,result,tasktype,resulttype)
|
177
|
+
end
|
178
|
+
|
179
|
+
def self.outstanding_tasks_template(iteration=nil)
|
180
|
+
fields.keys.sort.collect do |ii|
|
181
|
+
field = fields[ii]
|
182
|
+
case field
|
183
|
+
when :tasktype
|
184
|
+
:task
|
185
|
+
when :iteration
|
186
|
+
iteration
|
187
|
+
else
|
188
|
+
nil
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
def self.outstanding_results_template
|
194
|
+
fields.keys.sort.collect do |ii|
|
195
|
+
field = fields[ii]
|
196
|
+
case field
|
197
|
+
when :tasktype
|
198
|
+
:result
|
199
|
+
else
|
200
|
+
nil
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
def self.error_message(message,error)
|
206
|
+
result_message(message,error,:result,:error)
|
207
|
+
end
|
208
|
+
|
209
|
+
def error_message(error)
|
210
|
+
self.class.error_message(self,error)
|
211
|
+
end
|
212
|
+
|
213
|
+
def self.error_template(message)
|
214
|
+
fields.keys.sort.collect do |ii|
|
215
|
+
field = fields[ii]
|
216
|
+
case field
|
217
|
+
when :tasktype
|
218
|
+
message.tasktype
|
219
|
+
when :drburi, :version, :task_id
|
220
|
+
message.send(fields[ii])
|
221
|
+
else
|
222
|
+
nil
|
223
|
+
end
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
def error_template
|
228
|
+
self.class.error_template(self)
|
229
|
+
end
|
230
|
+
|
231
|
+
def self.fallback_task_message(message)
|
232
|
+
opts = Hash.new
|
233
|
+
fields.values.each do |field|
|
234
|
+
case field
|
235
|
+
when :iteration
|
236
|
+
opts[:iteration] = message.iteration + 1
|
237
|
+
when :expire_time
|
238
|
+
opts[:expire_time] = Time.now.to_i + message.expiry
|
239
|
+
else
|
240
|
+
opts[field] = message.send(field)
|
241
|
+
end
|
242
|
+
end
|
243
|
+
# debug "BUILDING NEXT FALLBACK TASK MESSAGE OFF"#, opts
|
244
|
+
Skynet::Message.new(opts)
|
245
|
+
end
|
246
|
+
|
247
|
+
def fallback_task_message
|
248
|
+
self.class.fallback_task_message(self)
|
249
|
+
end
|
250
|
+
|
251
|
+
def self.fallback_template(message)
|
252
|
+
fields.keys.sort.collect do |ii|
|
253
|
+
field = fields[ii]
|
254
|
+
case field
|
255
|
+
when :tasktype
|
256
|
+
message.tasktype
|
257
|
+
when :drburi, :version, :task_id
|
258
|
+
message.send(field)
|
259
|
+
when :iteration
|
260
|
+
(1..20)
|
261
|
+
else
|
262
|
+
nil
|
263
|
+
end
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
def fallback_template
|
268
|
+
self.class.fallback_template(self)
|
269
|
+
end
|
270
|
+
|
271
|
+
end ## END class Message
|
272
|
+
|
273
|
+
class WorkerVersionMessage < Skynet::Message
|
274
|
+
|
275
|
+
self.fields = self.superclass.fields
|
276
|
+
|
277
|
+
def initialize(opts)
|
278
|
+
super
|
279
|
+
self.expire_time ||= Time.now.to_i
|
280
|
+
self.tasktype = :current_worker_rev
|
281
|
+
end
|
282
|
+
|
283
|
+
def version
|
284
|
+
@version.to_i
|
285
|
+
end
|
286
|
+
|
287
|
+
def self.template
|
288
|
+
fields.keys.sort.collect do |ii|
|
289
|
+
field = fields[ii]
|
290
|
+
case field
|
291
|
+
when :tasktype
|
292
|
+
:current_worker_rev
|
293
|
+
else
|
294
|
+
nil
|
295
|
+
end
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
def template
|
300
|
+
fields.keys.sort.collect do |ii|
|
301
|
+
field = fields[ii]
|
302
|
+
case field
|
303
|
+
when :tasktype
|
304
|
+
:current_worker_rev
|
305
|
+
when :expire_time
|
306
|
+
nil
|
307
|
+
else
|
308
|
+
self.send(field)
|
309
|
+
end
|
310
|
+
end
|
311
|
+
end
|
312
|
+
end
|
313
|
+
|
314
|
+
class WorkerStatusMessage < Skynet::Message
|
315
|
+
self.fields = {
|
316
|
+
0 => :tasktype,
|
317
|
+
1 => :tasksubtype,
|
318
|
+
2 => :worker_id,
|
319
|
+
3 => :hostname,
|
320
|
+
4 => :process_id,
|
321
|
+
5 => :job_id,
|
322
|
+
6 => :task_id,
|
323
|
+
7 => :iteration,
|
324
|
+
8 => :name,
|
325
|
+
9 => :map_or_reduce,
|
326
|
+
10 => :started_at,
|
327
|
+
11 => :version,
|
328
|
+
12 => :processed
|
329
|
+
}
|
330
|
+
self.fields.values.each { |method| attr_accessor method }
|
331
|
+
|
332
|
+
def initialize(opts)
|
333
|
+
super
|
334
|
+
self.tasktype = :status
|
335
|
+
self.tasksubtype = :worker
|
336
|
+
end
|
337
|
+
|
338
|
+
def self.worker_status_template(opts)
|
339
|
+
fields.keys.sort.collect do |key|
|
340
|
+
case fields[key]
|
341
|
+
when :tasktype : :status
|
342
|
+
when :tasksubtype : :worker
|
343
|
+
when :hostname : opts[:hostname]
|
344
|
+
when :process_id : opts[:process_id]
|
345
|
+
else
|
346
|
+
nil
|
347
|
+
end
|
348
|
+
end
|
349
|
+
end
|
350
|
+
|
351
|
+
def self.all_workers_template(hostname=nil)
|
352
|
+
fields.keys.sort.collect do |key|
|
353
|
+
case fields[key]
|
354
|
+
when :tasktype : :status
|
355
|
+
when :tasksubtype : :worker
|
356
|
+
when :hostname
|
357
|
+
hostname if hostname
|
358
|
+
else
|
359
|
+
nil
|
360
|
+
end
|
361
|
+
end
|
362
|
+
end
|
363
|
+
|
364
|
+
end
|
365
|
+
|
366
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
class Skynet
|
2
|
+
|
3
|
+
class Error < StandardError
|
4
|
+
end
|
5
|
+
|
6
|
+
class ConnectionError < Skynet::Error
|
7
|
+
end
|
8
|
+
|
9
|
+
class QueueTimeout < Skynet::Error
|
10
|
+
end
|
11
|
+
|
12
|
+
class RequestExpiredError < Skynet::Error
|
13
|
+
end
|
14
|
+
|
15
|
+
class MessageQueue
|
16
|
+
|
17
|
+
include SkynetDebugger
|
18
|
+
|
19
|
+
require 'forwardable'
|
20
|
+
extend Forwardable
|
21
|
+
|
22
|
+
# require 'skynet_message'
|
23
|
+
|
24
|
+
def self.adapter
|
25
|
+
Object.module_eval(Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER], __FILE__, __LINE__).adapter
|
26
|
+
end
|
27
|
+
|
28
|
+
def initialize(message_queue_proxy_class=Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER])
|
29
|
+
if message_queue_proxy_class.is_a?(String)
|
30
|
+
@message_queue_proxy_class = Object.module_eval(message_queue_proxy_class, __FILE__, __LINE__)
|
31
|
+
else
|
32
|
+
@message_queue_proxy_class = message_queue_proxy_class
|
33
|
+
end
|
34
|
+
mq
|
35
|
+
end
|
36
|
+
|
37
|
+
def message_fields
|
38
|
+
Skynet::Message.fields
|
39
|
+
end
|
40
|
+
|
41
|
+
def mq
|
42
|
+
@mq ||= @message_queue_proxy_class.new
|
43
|
+
end
|
44
|
+
|
45
|
+
def_delegators :mq, :take_next_task, :write_message, :take_result, :write_error, :write_result,
|
46
|
+
:list_tasks, :list_results,
|
47
|
+
:clear_outstanding_tasks, :clear_outstanding_results,
|
48
|
+
:take_worker_status, :write_worker_status, :read_all_worker_statuses, :clear_worker_status,
|
49
|
+
:get_worker_version, :set_worker_version, :stats
|
50
|
+
|
51
|
+
|
52
|
+
|
53
|
+
|
54
|
+
def print_stats
|
55
|
+
"TAKEN TASKS: #{list_tasks(1).size}, UNTAKEN_TASKS: #{list_tasks(0).size} RESULTS: #{list_results.size}"
|
56
|
+
end
|
57
|
+
|
58
|
+
def list
|
59
|
+
list_tasks + list_results
|
60
|
+
end
|
61
|
+
|
62
|
+
def increment_worker_version
|
63
|
+
newver = self.get_worker_version + 1
|
64
|
+
self.set_worker_version(newver)
|
65
|
+
newver
|
66
|
+
end
|
67
|
+
|
68
|
+
def ansi_clear
|
69
|
+
puts "\033[2J\033[H"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
# def monitor_view
|
75
|
+
# ws = Skynet::WorkerStatusMessage.new([:status, :worker])
|
76
|
+
# ms = Skynet::ManagerStatusMessage.new([:status, :manager])
|
77
|
+
# local_stats = self.stats
|
78
|
+
# stattxt = []
|
79
|
+
# stattxt << "Last Update Time: #{Time.now}\n"
|
80
|
+
# stattxt << '%10s %10s %10s' % ["Untaken", "Taken", "Results"]
|
81
|
+
# stattxt << '%10d %10d %10d' % [local_stats[:untaken_tasks], local_stats[:taken_tasks], local_stats[:results]]
|
82
|
+
# stattxt << '%3s %-20s %10s %20s %20s %5s %5s %10s %s' % ['#', 'hostname', 'process_id', 'job_id', 'task_id', 'ver', 'proc', 'm/r', 'name' ]
|
83
|
+
# messages = mq.read_all(ws).collect{|tuple| Skynet::WorkerStatusMessage.new(tuple)}
|
84
|
+
# messages.sort!{ |a,b| a.process_id <=> b.process_id }
|
85
|
+
# messages.each_with_index do |message,index|
|
86
|
+
# stattxt << '%3d %-20s %10d %20d %20d %5d %5d %10s %s' % [
|
87
|
+
# index,
|
88
|
+
# message.hostname,
|
89
|
+
# message.process_id,
|
90
|
+
# message.job_id,
|
91
|
+
# message.task_id,
|
92
|
+
# message.version,
|
93
|
+
# message.processed,
|
94
|
+
# message.map_or_reduce || '-' ,
|
95
|
+
# message.name
|
96
|
+
# ]
|
97
|
+
# end
|
98
|
+
# return stattxt.join("\n")
|
99
|
+
# end
|
100
|
+
|