skynet 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. data/History.txt +4 -0
  2. data/License.txt +20 -0
  3. data/Manifest.txt +65 -0
  4. data/README.txt +100 -0
  5. data/Rakefile +4 -0
  6. data/app_generators/skynet_install/USAGE +5 -0
  7. data/app_generators/skynet_install/skynet_install_generator.rb +84 -0
  8. data/app_generators/skynet_install/templates/migration.rb +60 -0
  9. data/app_generators/skynet_install/templates/skynet +33 -0
  10. data/app_generators/skynet_install/templates/skynet_console +16 -0
  11. data/bin/skynet +20 -0
  12. data/bin/skynet_console +9 -0
  13. data/bin/skynet_install +12 -0
  14. data/bin/skynet_tuplespace_server +53 -0
  15. data/config/hoe.rb +74 -0
  16. data/config/requirements.rb +17 -0
  17. data/lib/skynet.rb +34 -0
  18. data/lib/skynet/mapreduce_test.rb +25 -0
  19. data/lib/skynet/message_queue_adapters/message_queue_adapter.rb +70 -0
  20. data/lib/skynet/message_queue_adapters/mysql.rb +573 -0
  21. data/lib/skynet/message_queue_adapters/tuple_space.rb +327 -0
  22. data/lib/skynet/skynet_active_record_extensions.rb +237 -0
  23. data/lib/skynet/skynet_config.rb +59 -0
  24. data/lib/skynet/skynet_console.rb +34 -0
  25. data/lib/skynet/skynet_console_helper.rb +59 -0
  26. data/lib/skynet/skynet_debugger.rb +84 -0
  27. data/lib/skynet/skynet_guid_generator.rb +68 -0
  28. data/lib/skynet/skynet_job.rb +607 -0
  29. data/lib/skynet/skynet_launcher.rb +10 -0
  30. data/lib/skynet/skynet_logger.rb +52 -0
  31. data/lib/skynet/skynet_manager.rb +486 -0
  32. data/lib/skynet/skynet_message.rb +366 -0
  33. data/lib/skynet/skynet_message_queue.rb +100 -0
  34. data/lib/skynet/skynet_ruby_extensions.rb +36 -0
  35. data/lib/skynet/skynet_task.rb +76 -0
  36. data/lib/skynet/skynet_tuplespace_server.rb +82 -0
  37. data/lib/skynet/skynet_worker.rb +395 -0
  38. data/lib/skynet/version.rb +9 -0
  39. data/log/debug.log +0 -0
  40. data/log/skynet.log +29 -0
  41. data/log/skynet_tuplespace_server.log +7 -0
  42. data/log/skynet_worker.pid +1 -0
  43. data/script/destroy +14 -0
  44. data/script/generate +14 -0
  45. data/script/txt2html +74 -0
  46. data/setup.rb +1585 -0
  47. data/sometest.rb +23 -0
  48. data/tasks/deployment.rake +34 -0
  49. data/tasks/environment.rake +7 -0
  50. data/tasks/website.rake +17 -0
  51. data/test/all_models_test.rb +139 -0
  52. data/test/mysql_message_queue_adaptor_test.rb +199 -0
  53. data/test/skynet_manager_test.rb +107 -0
  54. data/test/skynet_message_test.rb +42 -0
  55. data/test/test_generator_helper.rb +20 -0
  56. data/test/test_helper.rb +2 -0
  57. data/test/test_skynet.rb +11 -0
  58. data/test/test_skynet_install_generator.rb +53 -0
  59. data/test/tuplespace_message_queue_test.rb +179 -0
  60. data/tmtags +1242 -0
  61. data/website/index.html +93 -0
  62. data/website/index.txt +39 -0
  63. data/website/javascripts/rounded_corners_lite.inc.js +285 -0
  64. data/website/stylesheets/screen.css +138 -0
  65. data/website/template.rhtml +48 -0
  66. metadata +129 -0
@@ -0,0 +1,366 @@
1
+ class Skynet
2
+ class Message
3
+
4
+ include SkynetDebugger
5
+
6
+ class BadMessage < Skynet::Error
7
+ end
8
+
9
+ class << self
10
+ attr_accessor :fields
11
+ end
12
+
13
+ self.fields = {
14
+ 0 => :tasktype,
15
+ 1 => :drburi,
16
+ 2 => :task_id,
17
+ 3 => :job_id,
18
+ 4 => :payload,
19
+ 5 => :payload_type,
20
+ 6 => :name,
21
+ 7 => :expiry,
22
+ 8 => :expire_time,
23
+ 9 => :iteration,
24
+ 10 => :version
25
+ }
26
+
27
+ self.fields.values.each do |method|
28
+ next if method == :payload
29
+ next if method == :tasktype
30
+ next if method == :payload_type
31
+ attr_accessor method
32
+ end
33
+
34
+ attr_reader :payload_type, :tasktype
35
+
36
+ def initialize(opts)
37
+ if opts.is_a?(Array)
38
+ self.class.fields.each do |ii, field|
39
+ self.send("#{field}=",opts[ii] || nil)
40
+ end
41
+ elsif opts
42
+ self.class.fields.values.each do |field|
43
+ value = opts[field] || opts[field.to_s] || nil
44
+ self.send("#{field}=",value) if value
45
+ end
46
+ opts_raw_payload = opts[:raw_payload] || opts["raw_payload"]
47
+ if opts_raw_payload
48
+ self.raw_payload = opts_raw_payload
49
+ end
50
+ end
51
+ self.payload
52
+ end
53
+
54
+ def fields
55
+ self.class.fields
56
+ end
57
+
58
+ def tasktype=(ttype)
59
+ @tasktype = ttype.to_sym
60
+ end
61
+
62
+ def payload_type=(ptype)
63
+ @payload_type = ptype.to_sym if ptype
64
+ end
65
+
66
+ # alias for payload
67
+ def task
68
+ payload
69
+ end
70
+
71
+ def payload=(data)
72
+ @payload = data
73
+ self.raw_payload = data.to_yaml if data.respond_to?(:to_yaml) and not payload.kind_of?(Proc)
74
+ end
75
+
76
+ def payload
77
+ @payload ||= begin
78
+ YAML::load(self.raw_payload) if self.raw_payload
79
+ rescue Exception => e
80
+ raise BadMessage.new("Couldnt marshal payload #{e.inspect} #{e.backtrace.join("\n")}")
81
+ end
82
+ end
83
+
84
+ def raw_payload=(data)
85
+ @raw_payload = data
86
+ @payload=nil
87
+ end
88
+
89
+ def raw_payload
90
+ @raw_payload
91
+ end
92
+
93
+ def [](ii)
94
+ send(self.class.fields[ii])
95
+ end
96
+
97
+ def to_a
98
+ self.class.fields.keys.sort.collect do |ii|
99
+ self.send(self.class.fields[ii])
100
+ end
101
+ end
102
+
103
+ def to_hash
104
+ hash = {}
105
+ self.class.fields.keys.sort.collect do |ii|
106
+ hash[self.class.fields[ii]] = self.send(self.class.fields[ii])
107
+ end
108
+ hash
109
+ end
110
+
111
+ def to_h
112
+ to_hash
113
+ end
114
+
115
+ def to_s
116
+ to_a
117
+ end
118
+
119
+ def timeout
120
+ expire_time * 2
121
+ end
122
+
123
+ ####### TEMPLATES ############
124
+ def self.next_task_template(version=nil,payload_type=nil)
125
+ fields.keys.sort.collect do |ii|
126
+ field = fields[ii]
127
+ case field
128
+ when :expire_time
129
+ (0 .. Time.now.to_i)
130
+ when :tasktype
131
+ :task
132
+ when :version
133
+ version
134
+ when :payload_type
135
+ payload_type
136
+ when :iteration
137
+ (0..6)
138
+ else
139
+ nil
140
+ end
141
+ end
142
+ end
143
+
144
+ def self.result_template(job_id,tasktype=:result)
145
+ fields.keys.sort.collect do |ii|
146
+ field = fields[ii]
147
+ case field
148
+ when :tasktype
149
+ tasktype
150
+ when :job_id
151
+ job_id
152
+ else
153
+ nil
154
+ end
155
+ end
156
+ end
157
+
158
+ def self.result_message(message,result,tasktype=:result, resulttype=:result)
159
+ message_array = fields.keys.sort.collect do |ii|
160
+ field = fields[ii]
161
+ case field
162
+ when :tasktype
163
+ tasktype
164
+ when :payload
165
+ result
166
+ when :payload_type
167
+ resulttype
168
+ else
169
+ message.send(fields[ii])
170
+ end
171
+ end
172
+ new(message_array)
173
+ end
174
+
175
+ def result_message(result,tasktype=:result, resulttype=:result)
176
+ self.class.result_message(self,result,tasktype,resulttype)
177
+ end
178
+
179
+ def self.outstanding_tasks_template(iteration=nil)
180
+ fields.keys.sort.collect do |ii|
181
+ field = fields[ii]
182
+ case field
183
+ when :tasktype
184
+ :task
185
+ when :iteration
186
+ iteration
187
+ else
188
+ nil
189
+ end
190
+ end
191
+ end
192
+
193
+ def self.outstanding_results_template
194
+ fields.keys.sort.collect do |ii|
195
+ field = fields[ii]
196
+ case field
197
+ when :tasktype
198
+ :result
199
+ else
200
+ nil
201
+ end
202
+ end
203
+ end
204
+
205
+ def self.error_message(message,error)
206
+ result_message(message,error,:result,:error)
207
+ end
208
+
209
+ def error_message(error)
210
+ self.class.error_message(self,error)
211
+ end
212
+
213
+ def self.error_template(message)
214
+ fields.keys.sort.collect do |ii|
215
+ field = fields[ii]
216
+ case field
217
+ when :tasktype
218
+ message.tasktype
219
+ when :drburi, :version, :task_id
220
+ message.send(fields[ii])
221
+ else
222
+ nil
223
+ end
224
+ end
225
+ end
226
+
227
+ def error_template
228
+ self.class.error_template(self)
229
+ end
230
+
231
+ def self.fallback_task_message(message)
232
+ opts = Hash.new
233
+ fields.values.each do |field|
234
+ case field
235
+ when :iteration
236
+ opts[:iteration] = message.iteration + 1
237
+ when :expire_time
238
+ opts[:expire_time] = Time.now.to_i + message.expiry
239
+ else
240
+ opts[field] = message.send(field)
241
+ end
242
+ end
243
+ # debug "BUILDING NEXT FALLBACK TASK MESSAGE OFF"#, opts
244
+ Skynet::Message.new(opts)
245
+ end
246
+
247
+ def fallback_task_message
248
+ self.class.fallback_task_message(self)
249
+ end
250
+
251
+ def self.fallback_template(message)
252
+ fields.keys.sort.collect do |ii|
253
+ field = fields[ii]
254
+ case field
255
+ when :tasktype
256
+ message.tasktype
257
+ when :drburi, :version, :task_id
258
+ message.send(field)
259
+ when :iteration
260
+ (1..20)
261
+ else
262
+ nil
263
+ end
264
+ end
265
+ end
266
+
267
+ def fallback_template
268
+ self.class.fallback_template(self)
269
+ end
270
+
271
+ end ## END class Message
272
+
273
+ class WorkerVersionMessage < Skynet::Message
274
+
275
+ self.fields = self.superclass.fields
276
+
277
+ def initialize(opts)
278
+ super
279
+ self.expire_time ||= Time.now.to_i
280
+ self.tasktype = :current_worker_rev
281
+ end
282
+
283
+ def version
284
+ @version.to_i
285
+ end
286
+
287
+ def self.template
288
+ fields.keys.sort.collect do |ii|
289
+ field = fields[ii]
290
+ case field
291
+ when :tasktype
292
+ :current_worker_rev
293
+ else
294
+ nil
295
+ end
296
+ end
297
+ end
298
+
299
+ def template
300
+ fields.keys.sort.collect do |ii|
301
+ field = fields[ii]
302
+ case field
303
+ when :tasktype
304
+ :current_worker_rev
305
+ when :expire_time
306
+ nil
307
+ else
308
+ self.send(field)
309
+ end
310
+ end
311
+ end
312
+ end
313
+
314
+ class WorkerStatusMessage < Skynet::Message
315
+ self.fields = {
316
+ 0 => :tasktype,
317
+ 1 => :tasksubtype,
318
+ 2 => :worker_id,
319
+ 3 => :hostname,
320
+ 4 => :process_id,
321
+ 5 => :job_id,
322
+ 6 => :task_id,
323
+ 7 => :iteration,
324
+ 8 => :name,
325
+ 9 => :map_or_reduce,
326
+ 10 => :started_at,
327
+ 11 => :version,
328
+ 12 => :processed
329
+ }
330
+ self.fields.values.each { |method| attr_accessor method }
331
+
332
+ def initialize(opts)
333
+ super
334
+ self.tasktype = :status
335
+ self.tasksubtype = :worker
336
+ end
337
+
338
+ def self.worker_status_template(opts)
339
+ fields.keys.sort.collect do |key|
340
+ case fields[key]
341
+ when :tasktype : :status
342
+ when :tasksubtype : :worker
343
+ when :hostname : opts[:hostname]
344
+ when :process_id : opts[:process_id]
345
+ else
346
+ nil
347
+ end
348
+ end
349
+ end
350
+
351
+ def self.all_workers_template(hostname=nil)
352
+ fields.keys.sort.collect do |key|
353
+ case fields[key]
354
+ when :tasktype : :status
355
+ when :tasksubtype : :worker
356
+ when :hostname
357
+ hostname if hostname
358
+ else
359
+ nil
360
+ end
361
+ end
362
+ end
363
+
364
+ end
365
+
366
+ end
@@ -0,0 +1,100 @@
1
+ class Skynet
2
+
3
+ class Error < StandardError
4
+ end
5
+
6
+ class ConnectionError < Skynet::Error
7
+ end
8
+
9
+ class QueueTimeout < Skynet::Error
10
+ end
11
+
12
+ class RequestExpiredError < Skynet::Error
13
+ end
14
+
15
+ class MessageQueue
16
+
17
+ include SkynetDebugger
18
+
19
+ require 'forwardable'
20
+ extend Forwardable
21
+
22
+ # require 'skynet_message'
23
+
24
+ def self.adapter
25
+ Object.module_eval(Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER], __FILE__, __LINE__).adapter
26
+ end
27
+
28
+ def initialize(message_queue_proxy_class=Skynet::CONFIG[:MESSAGE_QUEUE_ADAPTER])
29
+ if message_queue_proxy_class.is_a?(String)
30
+ @message_queue_proxy_class = Object.module_eval(message_queue_proxy_class, __FILE__, __LINE__)
31
+ else
32
+ @message_queue_proxy_class = message_queue_proxy_class
33
+ end
34
+ mq
35
+ end
36
+
37
+ def message_fields
38
+ Skynet::Message.fields
39
+ end
40
+
41
+ def mq
42
+ @mq ||= @message_queue_proxy_class.new
43
+ end
44
+
45
+ def_delegators :mq, :take_next_task, :write_message, :take_result, :write_error, :write_result,
46
+ :list_tasks, :list_results,
47
+ :clear_outstanding_tasks, :clear_outstanding_results,
48
+ :take_worker_status, :write_worker_status, :read_all_worker_statuses, :clear_worker_status,
49
+ :get_worker_version, :set_worker_version, :stats
50
+
51
+
52
+
53
+
54
+ def print_stats
55
+ "TAKEN TASKS: #{list_tasks(1).size}, UNTAKEN_TASKS: #{list_tasks(0).size} RESULTS: #{list_results.size}"
56
+ end
57
+
58
+ def list
59
+ list_tasks + list_results
60
+ end
61
+
62
+ def increment_worker_version
63
+ newver = self.get_worker_version + 1
64
+ self.set_worker_version(newver)
65
+ newver
66
+ end
67
+
68
+ def ansi_clear
69
+ puts "\033[2J\033[H"
70
+ end
71
+ end
72
+ end
73
+
74
+ # def monitor_view
75
+ # ws = Skynet::WorkerStatusMessage.new([:status, :worker])
76
+ # ms = Skynet::ManagerStatusMessage.new([:status, :manager])
77
+ # local_stats = self.stats
78
+ # stattxt = []
79
+ # stattxt << "Last Update Time: #{Time.now}\n"
80
+ # stattxt << '%10s %10s %10s' % ["Untaken", "Taken", "Results"]
81
+ # stattxt << '%10d %10d %10d' % [local_stats[:untaken_tasks], local_stats[:taken_tasks], local_stats[:results]]
82
+ # stattxt << '%3s %-20s %10s %20s %20s %5s %5s %10s %s' % ['#', 'hostname', 'process_id', 'job_id', 'task_id', 'ver', 'proc', 'm/r', 'name' ]
83
+ # messages = mq.read_all(ws).collect{|tuple| Skynet::WorkerStatusMessage.new(tuple)}
84
+ # messages.sort!{ |a,b| a.process_id <=> b.process_id }
85
+ # messages.each_with_index do |message,index|
86
+ # stattxt << '%3d %-20s %10d %20d %20d %5d %5d %10s %s' % [
87
+ # index,
88
+ # message.hostname,
89
+ # message.process_id,
90
+ # message.job_id,
91
+ # message.task_id,
92
+ # message.version,
93
+ # message.processed,
94
+ # message.map_or_reduce || '-' ,
95
+ # message.name
96
+ # ]
97
+ # end
98
+ # return stattxt.join("\n")
99
+ # end
100
+