skynet 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. data/History.txt +4 -0
  2. data/License.txt +20 -0
  3. data/Manifest.txt +65 -0
  4. data/README.txt +100 -0
  5. data/Rakefile +4 -0
  6. data/app_generators/skynet_install/USAGE +5 -0
  7. data/app_generators/skynet_install/skynet_install_generator.rb +84 -0
  8. data/app_generators/skynet_install/templates/migration.rb +60 -0
  9. data/app_generators/skynet_install/templates/skynet +33 -0
  10. data/app_generators/skynet_install/templates/skynet_console +16 -0
  11. data/bin/skynet +20 -0
  12. data/bin/skynet_console +9 -0
  13. data/bin/skynet_install +12 -0
  14. data/bin/skynet_tuplespace_server +53 -0
  15. data/config/hoe.rb +74 -0
  16. data/config/requirements.rb +17 -0
  17. data/lib/skynet.rb +34 -0
  18. data/lib/skynet/mapreduce_test.rb +25 -0
  19. data/lib/skynet/message_queue_adapters/message_queue_adapter.rb +70 -0
  20. data/lib/skynet/message_queue_adapters/mysql.rb +573 -0
  21. data/lib/skynet/message_queue_adapters/tuple_space.rb +327 -0
  22. data/lib/skynet/skynet_active_record_extensions.rb +237 -0
  23. data/lib/skynet/skynet_config.rb +59 -0
  24. data/lib/skynet/skynet_console.rb +34 -0
  25. data/lib/skynet/skynet_console_helper.rb +59 -0
  26. data/lib/skynet/skynet_debugger.rb +84 -0
  27. data/lib/skynet/skynet_guid_generator.rb +68 -0
  28. data/lib/skynet/skynet_job.rb +607 -0
  29. data/lib/skynet/skynet_launcher.rb +10 -0
  30. data/lib/skynet/skynet_logger.rb +52 -0
  31. data/lib/skynet/skynet_manager.rb +486 -0
  32. data/lib/skynet/skynet_message.rb +366 -0
  33. data/lib/skynet/skynet_message_queue.rb +100 -0
  34. data/lib/skynet/skynet_ruby_extensions.rb +36 -0
  35. data/lib/skynet/skynet_task.rb +76 -0
  36. data/lib/skynet/skynet_tuplespace_server.rb +82 -0
  37. data/lib/skynet/skynet_worker.rb +395 -0
  38. data/lib/skynet/version.rb +9 -0
  39. data/log/debug.log +0 -0
  40. data/log/skynet.log +29 -0
  41. data/log/skynet_tuplespace_server.log +7 -0
  42. data/log/skynet_worker.pid +1 -0
  43. data/script/destroy +14 -0
  44. data/script/generate +14 -0
  45. data/script/txt2html +74 -0
  46. data/setup.rb +1585 -0
  47. data/sometest.rb +23 -0
  48. data/tasks/deployment.rake +34 -0
  49. data/tasks/environment.rake +7 -0
  50. data/tasks/website.rake +17 -0
  51. data/test/all_models_test.rb +139 -0
  52. data/test/mysql_message_queue_adaptor_test.rb +199 -0
  53. data/test/skynet_manager_test.rb +107 -0
  54. data/test/skynet_message_test.rb +42 -0
  55. data/test/test_generator_helper.rb +20 -0
  56. data/test/test_helper.rb +2 -0
  57. data/test/test_skynet.rb +11 -0
  58. data/test/test_skynet_install_generator.rb +53 -0
  59. data/test/tuplespace_message_queue_test.rb +179 -0
  60. data/tmtags +1242 -0
  61. data/website/index.html +93 -0
  62. data/website/index.txt +39 -0
  63. data/website/javascripts/rounded_corners_lite.inc.js +285 -0
  64. data/website/stylesheets/screen.css +138 -0
  65. data/website/template.rhtml +48 -0
  66. metadata +129 -0
@@ -0,0 +1,53 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rubygems'
4
+ require 'daemons'
5
+ require 'pp'
6
+
7
+ require File.expand_path(File.dirname(__FILE__)) + '/../lib/skynet.rb'
8
+
9
+ options = {
10
+ :port => 7647,
11
+ :logfile => Skynet::CONFIG[:SKYNET_LOG_FILE],
12
+ :loglevel => "DEBUG",
13
+ :piddir => Skynet::CONFIG[:SKYNET_PID_DIR]
14
+ }
15
+
16
+ OptionParser.new do |opt|
17
+ opt.banner = "Usage: skynet_tuplespace_server (start|stop|run) [options]"
18
+ opt.on('-t', '--ontop TRUE', 'Dont Daemonize') do |v|
19
+ options[:ontop] = true if v.upcase == "TRUE" or v == "1"
20
+ end
21
+ opt.on('-p', '--port PORT', 'Port to listen on. default 7647') do |v|
22
+ options[:port] = v.to_i
23
+ end
24
+ opt.on('-o', '--log LOGFILE', 'Logfile to log to') do |v|
25
+ options[:logfile] = v
26
+ end
27
+ opt.on('-l', '--loglevel LOGLEVEL', 'Log level defaults to DEBUG') do |v|
28
+ options[:loglevel] = v
29
+ end
30
+ opt.on('-d', '--piddir PIDDIR', 'Directory to put pidfile') do |v|
31
+ options[:piddir] = File.expand_path(v)
32
+ end
33
+ opt.on('-u', '--drburi Drb URI', 'What DRbURI to use') do |v|
34
+ if v =~ %r{druby://}
35
+ options[:drburi] = v
36
+ else
37
+ options[:drburi] = "druby://#{v}"
38
+ end
39
+ end
40
+
41
+ opt.parse!(ARGV)
42
+ end
43
+
44
+ Daemons.run_proc("skynet_tuplespace_server#{options[:port]}",
45
+ {
46
+ :dir_mode => :normal,
47
+ :dir => options[:piddir],
48
+ :backtrace => true,
49
+ # :monitor => true,
50
+ :ontop => options[:ontop] || false
51
+ }) do
52
+ server = Skynet::Server.new(options)
53
+ end
@@ -0,0 +1,74 @@
1
+ require 'skynet/version'
2
+
3
+ AUTHOR = 'Adam Pisoni' # can also be an array of Authors
4
+ EMAIL = "apisoni@geni.com"
5
+ DESCRIPTION = "Skynet - A Ruby Map/Reduce Framework"
6
+ GEM_NAME = 'skynet' # what ppl will type to install your gem
7
+ RUBYFORGE_PROJECT = 'skynet' # The unix name for your project
8
+ HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
9
+ DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
10
+
11
+ @config_file = "~/.rubyforge/user-config.yml"
12
+ @config = nil
13
+ RUBYFORGE_USERNAME = "unknown"
14
+ def rubyforge_username
15
+ unless @config
16
+ begin
17
+ @config = YAML.load(File.read(File.expand_path(@config_file)))
18
+ rescue
19
+ puts <<-EOS
20
+ ERROR: No rubyforge config file found: #{@config_file}
21
+ Run 'rubyforge setup' to prepare your env for access to Rubyforge
22
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
23
+ EOS
24
+ exit
25
+ end
26
+ end
27
+ RUBYFORGE_USERNAME.replace @config["username"]
28
+ end
29
+
30
+
31
+ REV = nil
32
+ # UNCOMMENT IF REQUIRED:
33
+ # REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
34
+ VERS = Skynet::VERSION::STRING + (REV ? ".#{REV}" : "")
35
+ RDOC_OPTS = ['--quiet', '--title', 'skynet documentation',
36
+ "--opname", "index.html",
37
+ "--line-numbers",
38
+ "--main", "README",
39
+ "--inline-source"]
40
+
41
+ class Hoe
42
+ def extra_deps
43
+ @extra_deps.reject! { |x| Array(x).first == 'hoe' }
44
+ @extra_deps
45
+ end
46
+ end
47
+
48
+ # Generate all the Rake tasks
49
+ # Run 'rake -T' to see list of generated tasks (from gem root directory)
50
+ hoe = Hoe.new(GEM_NAME, VERS) do |p|
51
+ p.author = AUTHOR
52
+ p.description = DESCRIPTION
53
+ p.email = EMAIL
54
+ p.summary = DESCRIPTION
55
+ p.url = HOMEPATH
56
+ p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
57
+ p.test_globs = ["test/**/test_*.rb"]
58
+ p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store'] #An array of file patterns to delete on clean.
59
+
60
+ # == Optional
61
+ p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
62
+ p.extra_deps = [
63
+ ['daemons',">= 1"],
64
+ ]
65
+ # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
66
+
67
+ #p.spec_extras = {} # A hash of extra values to set in the gemspec.
68
+
69
+ end
70
+
71
+ CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
72
+ PATH = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
73
+ hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
74
+ hoe.rsync_args = '-av --delete --ignore-errors'
@@ -0,0 +1,17 @@
1
+ require 'fileutils'
2
+ include FileUtils
3
+
4
+ require 'rubygems'
5
+ %w[rake hoe newgem rubigen].each do |req_gem|
6
+ begin
7
+ require req_gem
8
+ rescue LoadError
9
+ puts "This Rakefile requires the '#{req_gem}' RubyGem."
10
+ puts "Installation: gem install #{req_gem} -y"
11
+ exit
12
+ end
13
+ end
14
+
15
+ $:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
16
+
17
+ require 'skynet'
@@ -0,0 +1,34 @@
1
+ $:.unshift File.dirname(__FILE__)
2
+ $:.unshift File.dirname(__FILE__) + '/skynet'
3
+
4
+ # path = File.expand_path(File.dirname(__FILE__))
5
+
6
+
7
+ require 'drb'
8
+ require 'skynet_guid_generator'
9
+ require 'skynet_logger'
10
+ require 'skynet_config'
11
+
12
+ Skynet::CONFIG[:SKYNET_PATH] ||= File.expand_path(File.dirname(__FILE__) +"/..")
13
+ # Skynet::CONFIG[:LAUNCHER_PATH] ||= File.expand_path(ENV['_'])
14
+
15
+ require 'skynet_debugger'
16
+ require 'skynet_message'
17
+ require 'message_queue_adapters/message_queue_adapter'
18
+ require 'message_queue_adapters/tuple_space'
19
+ require "skynet_message_queue"
20
+ require 'skynet_job'
21
+ require 'skynet_worker'
22
+ require 'skynet_task'
23
+ require 'skynet_manager'
24
+ require 'skynet_tuplespace_server'
25
+ require 'skynet_ruby_extensions'
26
+ begin
27
+ require 'active_record'
28
+ require 'skynet_active_record_extensions'
29
+ require 'message_queue_adapters/mysql'
30
+ rescue LoadError => e
31
+ end
32
+ require 'mapreduce_test'
33
+ require 'skynet_launcher'
34
+ require 'skynet_console'
@@ -0,0 +1,25 @@
1
+ class Skynet
2
+ class MapreduceTest
3
+ include SkynetDebugger
4
+
5
+ def self.map(datas)
6
+ results = {}
7
+ datas.each do |data|
8
+ results[data] ||= 0
9
+ results[data] += 1
10
+ end
11
+ [results]
12
+ end
13
+
14
+ def self.reduce(datas)
15
+ results = {}
16
+ datas.each do |hashes|
17
+ hashes.each do |key,value|
18
+ results[key] ||= 0
19
+ results[key] += value
20
+ end
21
+ end
22
+ results
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,70 @@
1
+ class Skynet
2
+
3
+ class Error < StandardError
4
+ end
5
+
6
+ class RequestExpiredError < Skynet::Error
7
+ end
8
+
9
+ class InvalidMessage < Skynet::Error
10
+ end
11
+
12
+ class AbstractClassError < Skynet::Error
13
+ end
14
+
15
+ class MessageQueueAdapter
16
+
17
+ def list_results(data,timeout=nil)
18
+ raise AbstractClassError.new("You must implement list_results in a subclass.")
19
+ end
20
+
21
+ def list_tasks(template,timeout=nil)
22
+ raise AbstractClassError.new("You must implement method in a subclass.")
23
+ end
24
+
25
+ def take_next_task(template,timeout=nil)
26
+ raise AbstractClassError.new("You must implement method in a subclass.")
27
+ end
28
+
29
+ def write_message(template,timeout=nil)
30
+ raise AbstractClassError.new("You must implement method in a subclass.")
31
+ end
32
+
33
+ def write_result(template,timeout=nil)
34
+ raise AbstractClassError.new("You must implement method in a subclass.")
35
+ end
36
+
37
+ def take_result(template,timeout=nil)
38
+ raise AbstractClassError.new("You must implement method in a subclass.")
39
+ end
40
+
41
+ def write_error(template,timeout=nil)
42
+ raise AbstractClassError.new("You must implement method in a subclass.")
43
+ end
44
+
45
+ def write_worker_status(template,timeout=nil)
46
+ raise AbstractClassError.new("You must implement method in a subclass.")
47
+ end
48
+
49
+ def take_worker_status(template,timeout=nil)
50
+ raise AbstractClassError.new("You must implement method in a subclass.")
51
+ end
52
+
53
+ def read_all_worker_statuses(template,timeout=nil)
54
+ raise AbstractClassError.new("You must implement method in a subclass.")
55
+ end
56
+
57
+ def get_worker_version(template,timeout=nil)
58
+ raise AbstractClassError.new("You must implement method in a subclass.")
59
+ end
60
+
61
+ def set_worker_version(template,timeout=nil)
62
+ raise AbstractClassError.new("You must implement method in a subclass.")
63
+ end
64
+
65
+ def clear_outstanding_tasks
66
+ raise AbstractClassError.new("You must implement clear_outstanding_tasks in a subclass.")
67
+ end
68
+
69
+ end
70
+ end
@@ -0,0 +1,573 @@
1
+ class SkynetMessageQueue < ActiveRecord::Base
2
+ end
3
+
4
+ class SkynetWorkerQueue < ActiveRecord::Base
5
+ end
6
+
7
+
8
+ class Skynet
9
+
10
+ # require 'mysql'
11
+
12
+ class Error < StandardError
13
+ end
14
+
15
+ class RequestExpiredError < Skynet::Error
16
+ end
17
+
18
+ class InvalidMessage < Skynet::Error
19
+ end
20
+
21
+ class MessageQueueAdapter
22
+
23
+ class Mysql < Skynet::MessageQueueAdapter
24
+
25
+ include SkynetDebugger
26
+ include Skynet::GuidGenerator
27
+
28
+ SEARCH_FIELDS = [:tasktype, :task_id, :job_id, :payload_type, :expire_time, :iteration, :version] unless defined?(SEARCH_FIELDS)
29
+
30
+ Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY] ||= 30
31
+
32
+
33
+ @@db_set = false
34
+
35
+ def self.adapter
36
+ :mysql
37
+ end
38
+
39
+ def initialize
40
+ if Skynet::CONFIG[:QUEUE_DATABASE] and not @@db_set
41
+ begin
42
+ SkynetMessageQueue.establish_connection Skynet::CONFIG[:QUEUE_DATABASE]
43
+ SkynetWorkerQueue.establish_connection Skynet::CONFIG[:QUEUE_DATABASE]
44
+ rescue ActiveRecord::AdapterNotSpecified => e
45
+ warn "#{Skynet::CONFIG[:QUEUE_DATABASE]} not defined as a database adaptor #{e.message}"
46
+ end
47
+ end
48
+ @@db_set = true
49
+ end
50
+
51
+ # def initialize
52
+ # SkynetMessageQueue.connection.execute("set session TRANSACTION ISOLATION LEVEL READ UNCOMMITTED")
53
+ # end
54
+
55
+ def self.debug_class_desc
56
+ "MYSQLMQ"
57
+ end
58
+
59
+ def message_to_conditions(message)
60
+ template_to_conditions(message.to_a)
61
+ end
62
+
63
+ def template_to_conditions(template,fields=Skynet::Message.fields)
64
+ fields = fields.invert
65
+ conditions = []
66
+ values = []
67
+
68
+ fields.keys.each do |field|
69
+ value = template[fields[field]]
70
+ next unless value
71
+ if value.is_a?(Range)
72
+ conditions << "#{field} BETWEEN #{value.first} AND #{value.last}"
73
+ elsif value.is_a?(Symbol) or value.is_a?(String)
74
+ conditions << "#{field} = '#{value}'"
75
+ else
76
+ conditions << "#{field} = #{value}"
77
+ end
78
+ end
79
+ return '' if conditions.empty?
80
+ return conditions.join(" AND ")
81
+ end
82
+
83
+ def message_to_hash(message,timeout=nil,fields=Skynet::Message.fields)
84
+ hash = {}
85
+ fields.values.each do |field|
86
+ next if field == :drburi
87
+ # next unless message.send(field)
88
+ if message.send(field).is_a?(Symbol)
89
+ hash[field] = message.send(field).to_s
90
+ elsif field == :payload
91
+ hash[:raw_payload] = message.raw_payload
92
+ else
93
+ hash[field] = message.send(field)
94
+ end
95
+ end
96
+ if timeout
97
+ hash[:timeout] = timeout
98
+ hash[:expire_time] = (Time.now.to_f + timeout) unless hash[:expire_time]
99
+ end
100
+ hash
101
+ end
102
+
103
+ def take_next_task(curver,timeout=0,payload_type=nil)
104
+ timeout = Skynet::CONFIG[:NEXT_TASK_TIMEOUT] if timeout < 1
105
+ debug "TASK NEXT TASK!!!!!!! timeout: #{timeout}"
106
+ message = nil
107
+ start = Time.now
108
+ rows = nil
109
+ loop do
110
+ # debug "start #{Time.now} timeout #{start + timeout}"
111
+ message_row = take(Skynet::Message.next_task_template(curver,payload_type),start,timeout)
112
+ next unless message_row
113
+
114
+ begin
115
+ message = Skynet::Message.new(message_row.attributes)
116
+ rescue Skynet::Message::BadMessage => e
117
+ message = nil
118
+ message_row.destroy
119
+ next
120
+ end
121
+ ftm = message.fallback_task_message
122
+ rows = update("update skynet_message_queues set iteration = #{ftm.iteration }, expire_time = #{ftm.expire_time} where iteration = #{message.iteration} and id = #{message_row.id}")
123
+
124
+ # message = nil if rows == 0
125
+ return message if message
126
+ end
127
+ end
128
+
129
+
130
+
131
+ def write_message(message,timeout=nil)
132
+ SkynetMessageQueue.create(message_to_hash(message, timeout))
133
+ end
134
+
135
+ def write_result(message,result=[],timeout=nil)
136
+ result_message = message.result_message(result)
137
+ result_message.expire_time = nil
138
+ update_message(result_message,timeout)
139
+ end
140
+
141
+ def update_message(message,timeout=nil)
142
+ timeout_sql = (timeout ? ", timeout = #{timeout}, expire_time = #{Time.now.to_f + timeout}" : '')
143
+ rows = 0
144
+ rows = update(%{
145
+ update skynet_message_queues
146
+ set tasktype = "#{message.tasktype}",
147
+ raw_payload = '#{message.raw_payload}',
148
+ payload_type = "#{message.payload_type}",
149
+ tran_id = NULL
150
+ #{timeout_sql}
151
+ where task_id = #{message.task_id}
152
+ })
153
+
154
+ raise Skynet::RequestExpiredError.new() if rows == 0
155
+ end
156
+
157
+ def take_result(job_id,timeout=1)
158
+ start = Time.now
159
+ result = nil
160
+ sleep_time = 10
161
+ if timeout < 1
162
+ sleep_time = 1
163
+ elsif timeout > 10
164
+ sleep_time = 10
165
+ else
166
+ sleep_time = timeout * 0.25
167
+ end
168
+ message_row = nil
169
+
170
+ loop do
171
+ # message_row = take(Skynet::Message.result_template(job_id), start, timeout,sleep_time)
172
+ conditions = template_to_conditions(Skynet::Message.result_template(job_id))
173
+ # sleep_time ||= timeout
174
+
175
+ message_row = SkynetMessageQueue.find(:first,:conditions => conditions)
176
+ break if message_row
177
+
178
+ if Time.now.to_f > start.to_f + timeout
179
+ raise Skynet::RequestExpiredError.new
180
+ else
181
+ sleepy = rand(sleep_time)
182
+ # error "RESULT EMPTY SLEEPING: #{sleepy}"
183
+ sleep sleepy
184
+ next
185
+ end
186
+ next
187
+ end
188
+
189
+ result = Skynet::Message.new(message_row.clone.attributes)
190
+ message_row.destroy
191
+ return result if result
192
+ end
193
+
194
+ def list_tasks(iteration=nil)
195
+ conditions = template_to_conditions(Skynet::Message.outstanding_tasks_template(iteration))
196
+ SkynetMessageQueue.find(:all,:conditions => conditions)
197
+ end
198
+
199
+ def list_results
200
+ conditions = template_to_conditions(Skynet::Message.outstanding_results_template)
201
+ SkynetMessageQueue.find(:all,:conditions => conditions)
202
+ end
203
+
204
+ def write_error(message,error='',timeout=nil)
205
+ message.expire_time = nil
206
+ update_message(message.error_message(error),timeout)
207
+ end
208
+
209
+ def write_worker_status(task, timeout=nil)
210
+ message = Skynet::WorkerStatusMessage.new(task)
211
+ worker_fields = Skynet::WorkerStatusMessage.fields.reject {|k,f| f == :process_id or f == :hostname or f == :tasksubtype or f == :tasktype}
212
+ update_hash = message_to_hash(message, timeout, Skynet::WorkerStatusMessage.fields)
213
+ update_hash.each do |k,v|
214
+ if not v
215
+ update_hash[k] = "NULL"
216
+ elsif v.kind_of?(String) or v.kind_of?(Symbol)
217
+ update_hash[k] = "'#{v}'"
218
+ end
219
+ end
220
+
221
+ update_sql = "UPDATE skynet_worker_queues SET #{update_hash.collect{|k,v| "#{k}=#{v}"}.join(',')} WHERE worker_id=#{message.worker_id}"
222
+ rows = update(update_sql)
223
+ if rows == 0
224
+ begin
225
+ insert_sql = "INSERT INTO skynet_worker_queues (#{update_hash.keys.join(',')}) VALUES (#{update_hash.values.join(',')})"
226
+ rows = update(insert_sql)
227
+ rescue ActiveRecord::StatementInvalid => e
228
+ if e.message =~ /Duplicate/
229
+ error "DUPLICATE WORKER #{e.message}"
230
+ else
231
+ raise e
232
+ end
233
+ end
234
+ end
235
+ return rows
236
+ end
237
+
238
+ def take_worker_status(task, timeout=nil)
239
+ conditions = template_to_conditions(Skynet::WorkerStatusMessage.worker_status_template(task), Skynet::WorkerStatusMessage.fields)
240
+ worker_status = nil
241
+ SkynetWorkerQueue.transaction do
242
+ worker_row = SkynetWorkerQueue.find(:first, :conditions => conditions)
243
+ return unless worker_row
244
+ worker_status = Skynet::WorkerStatusMessage.new(worker_row.clone.attributes)
245
+ worker_row.destroy
246
+ end
247
+ worker_status
248
+ end
249
+
250
+ def read_all_worker_statuses(hostname=nil,process_id=nil)
251
+ ws = Skynet::WorkerStatusMessage.all_workers_template(hostname)
252
+ ws[4] = process_id if process_id
253
+ conditions = template_to_conditions(ws,Skynet::WorkerStatusMessage.fields)
254
+ rows = SkynetWorkerQueue.find(:all, :conditions => conditions)
255
+ workers = rows.collect{ |w| Skynet::WorkerStatusMessage.new(w.attributes) }#.sort{ |a,b| a.process_id <=> b.process_id }
256
+ end
257
+
258
+
259
+ def clear_worker_status(hostname=nil)
260
+ if hostname
261
+ SkynetWorkerQueue.connection.execute("delete from skynet_worker_queues where hostname = '#{hostname}'")
262
+ else
263
+ SkynetWorkerQueue.destroy_all
264
+ end
265
+ end
266
+
267
+ def set_worker_version(ver=nil)
268
+ ver ||= 1
269
+ # SkynetWorkerQueue.transaction do
270
+ # SkynetWorkerQueue.connection.execute("delete from skynet_worker_queues where tasktype = 'workerversion'")
271
+ SkynetWorkerQueue.connection.insert("replace skynet_worker_queues (worker_id, tasktype, version) values (0, 'workerversion',#{ver})")
272
+ # end
273
+ ver
274
+ end
275
+
276
+ def get_worker_version
277
+ # ver = SkynetWorkerQueue.connection.select_value("select min(version) from skynet_message_queues where tasktype = 'task'")
278
+ ver = SkynetWorkerQueue.connection.select_value("select version from skynet_worker_queues where tasktype = 'workerversion'")
279
+ if not ver
280
+ set_worker_version(1)
281
+ ver = 1
282
+ end
283
+ ver.to_i
284
+ end
285
+
286
+
287
+ def clear_outstanding_tasks
288
+ SkynetMessageQueue.destroy_all
289
+ end
290
+
291
+ def delete_expired_messages
292
+ SkynetMessageQueue.connection.delete("delete from skynet_message_queues where expire_time BETWEEN 1 AND '#{Time.now.to_f}'")
293
+ end
294
+
295
+ # select hostname, iteration, count(id) as number_of_workers, count(iteration) as iteration, sum(processed) as processed, max(started_at) as most_recent_task_time from skynet_worker_queues where tasksubtype = 'worker' group by hostname, iteration;
296
+ #
297
+ # select hostname, count(id) as number_of_workers, sum(processed) as processed, max(started_at) as most_recent_task_time,
298
+ # CASE iteration WHEN NULL
299
+ #
300
+ # from skynet_worker_queues where tasksubtype = 'worker' group by hostname;
301
+
302
+ def stats
303
+ stats = {
304
+ :servers => {},
305
+ :results => 0,
306
+ :taken_tasks => 0,
307
+ :untaken_tasks => 0,
308
+ :taken_master_tasks => 0,
309
+ :taken_task_tasks => 0,
310
+ :untaken_master_tasks => 0,
311
+ :untaken_task_tasks => 0,
312
+ :processed => 0,
313
+ :number_of_workers => 0,
314
+ :active_workers => 0,
315
+ :idle_workers => 0,
316
+ :hosts => 0,
317
+ :masters => 0,
318
+ :taskworkers => 0,
319
+ :time => Time.now.to_f
320
+ }
321
+
322
+ stat_rows = SkynetWorkerQueue.connection.select_all(%{
323
+ SELECT tasktype, payload_type, iteration, count(id) as number_of_tasks
324
+ FROM skynet_message_queues
325
+ GROUP BY tasktype, payload_type, iteration
326
+ })
327
+ # pp stat_rows
328
+ stat_rows.each do |row|
329
+ if row["tasktype"] == "result" or row["payload_type"] == "result"
330
+ stats[:results] += row["number_of_tasks"].to_i
331
+ elsif row["tasktype"] == "task"
332
+ type_of_tasks = nil
333
+ if row["payload_type"] == "master"
334
+ type_of_tasks = :master_tasks
335
+ elsif row["payload_type"] == "task"
336
+ type_of_tasks = :task_tasks
337
+ end
338
+ if row["iteration"].to_i > 0
339
+ stats["taken_#{type_of_tasks}".to_sym] += row["number_of_tasks"].to_i
340
+ stats[:taken_tasks] += row["number_of_tasks"].to_i
341
+ else
342
+ stats["untaken_#{type_of_tasks}".to_sym] += row["number_of_tasks"].to_i
343
+ stats[:untaken_tasks] += row["number_of_tasks"].to_i
344
+ end
345
+ end
346
+ end
347
+
348
+ servers = {}
349
+
350
+ stat_sql = <<-SQL
351
+ select hostname, map_or_reduce, count(id) number_of_workers, sum(processed) as processed,
352
+ max(started_at) as most_recent_task_time, iteration
353
+ FROM skynet_worker_queues
354
+ WHERE skynet_worker_queues.tasksubtype = 'worker'
355
+ SQL
356
+
357
+ stat_rows = SkynetWorkerQueue.connection.select_all("#{stat_sql} GROUP BY hostname, map_or_reduce").each do |row|
358
+ servers[row["hostname"]] ||= {
359
+ :processed => 0,
360
+ :hostname => row["hostname"],
361
+ :number_of_workers => 0,
362
+ :active_workers => 0,
363
+ :idle_workers => 0,
364
+ }
365
+
366
+ servers[row["hostname"]][:processed] += row["processed"].to_i
367
+ servers[row["hostname"]][:number_of_workers] += row["number_of_workers"].to_i
368
+ servers[row["hostname"]][:active_workers] += 0
369
+ servers[row["hostname"]][:idle_workers] += row["number_of_workers"].to_i
370
+ stats[:processed] += row["processed"].to_i
371
+ stats[:number_of_workers] += row["number_of_workers"].to_i
372
+ stats[:idle_workers] += row["number_of_workers"].to_i
373
+ end
374
+
375
+ SkynetWorkerQueue.connection.select_all(%{
376
+ #{stat_sql} AND skynet_worker_queues.iteration IS NOT NULL
377
+ GROUP BY hostname, map_or_reduce
378
+ }).each do |row|
379
+ map_or_reduce = nil
380
+ if row["map_or_reduce"] == "master"
381
+ map_or_reduce = :masters
382
+ else
383
+ map_or_reduce = :taskworkers
384
+ end
385
+ servers[row["hostname"]][:active_workers] += row["number_of_workers"].to_i
386
+ servers[row["hostname"]][:idle_workers] -= row["number_of_workers"].to_i
387
+ servers[row["hostname"]][map_or_reduce] ||= 0
388
+ servers[row["hostname"]][map_or_reduce] += row["number_of_workers"].to_i
389
+ stats[map_or_reduce] += row["number_of_workers"].to_i
390
+ stats[:active_workers] += row["number_of_workers"].to_i
391
+ stats[:idle_workers] -= row["number_of_workers"].to_i
392
+ end
393
+
394
+ stats[:servers] = servers
395
+ stats[:hosts] = servers.keys.size
396
+ stats[:time] = Time.now.to_f - stats[:time]
397
+ stats
398
+ end
399
+
400
+ def processed(sleepy=5,tim=10)
401
+ last_time = Time.now
402
+ last_count = Skynet::MessageQueue.new.stats[:processed]
403
+ tim.times do
404
+ new_count = Skynet::MessageQueue.new.stats[:processed]
405
+ new_time = Time.now
406
+ puts "Processed #{new_count - last_count} in #{new_time - last_time}"
407
+ last_time = new_time
408
+ last_count = new_count
409
+ sleep sleepy
410
+ end
411
+ end
412
+
413
+ private
414
+
415
+ def update(sql)
416
+ rows = 0
417
+ 3.times do
418
+ begin
419
+ rows = SkynetMessageQueue.connection.update(sql)
420
+ return rows
421
+ rescue ActiveRecord::StatementInvalid => e
422
+ if e.message =~ /Deadlock/ or e.message =~ /Transaction/
423
+ error "#{self.class} update had collision #{e.message}"
424
+ sleep 0.1
425
+ next
426
+ else
427
+ raise e
428
+ end
429
+ end
430
+ end
431
+ return rows
432
+ end
433
+
434
+ Skynet::CONFIG[:MYSQL_TEMPERATURE_CHANGE_SLEEP] ||= 40
435
+
436
+ @@temperature ||= {}
437
+ @@temperature[:task] ||= 1
438
+ @@temperature[:master] ||= 1
439
+ @@temperature[:any] ||= 1
440
+
441
+ def take(template,start=Time.now,timeout=1,sleep_time=nil)
442
+ conditions = template_to_conditions(template)
443
+ sleep_time ||= timeout
444
+ transaction_id = get_unique_id(1)
445
+ times_tried = 0
446
+ payload_type = template[Skynet::Message.fields.invert[:payload_type]]
447
+ payload_type ||= :any
448
+ payload_type = payload_type.to_sym
449
+
450
+ 10.times do
451
+ begin
452
+ ## TEPERATURE
453
+ temperature_sql = (temperature(payload_type) > 1 ? " AND id % #{temperature(payload_type).ceil} = #{rand(temperature(payload_type)).to_i} " : '')
454
+
455
+ ### Mqke sure we get the old ones. If we order by on ever select its VERY expensive.
456
+ order_by = (payload_type != :master and rand(100) < 5) ? "ORDER BY payload_type desc, created_on desc" : ''
457
+
458
+ sql = <<-SQL
459
+ SELECT *
460
+ FROM skynet_message_queues
461
+ WHERE #{conditions} #{temperature_sql}
462
+ #{order_by}
463
+ LIMIT 1
464
+ SQL
465
+
466
+ message_row = SkynetMessageQueue.find_by_sql(sql).first
467
+ if message_row
468
+ update_conditions = "ID = #{message_row.id} and tran_id "
469
+ if message_row.tran_id
470
+ update_conditions << "= #{message_row.tran_id}"
471
+ else
472
+ update_conditions << 'IS NULL'
473
+ end
474
+ rows = SkynetMessageQueue.connection.update(
475
+ "UPDATE skynet_message_queues set tran_id = #{transaction_id} WHERE #{update_conditions}"
476
+ )
477
+ if rows < 1
478
+ old_temp = temperature(payload_type)
479
+ set_temperature(payload_type,conditions)
480
+ info "MISSCOLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
481
+ next
482
+ end
483
+ return message_row
484
+ else
485
+ old_temp = temperature(payload_type)
486
+ set_temperature(payload_type,conditions)
487
+ info "MISS PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
488
+ break if temperature(payload_type) == 1 and old_temp == 1
489
+ next
490
+ end
491
+ rescue ActiveRecord::StatementInvalid => e
492
+ if e.message =~ /Deadlock/
493
+ old_temp = temperature(payload_type)
494
+ set_temperature(payload_type,conditions)
495
+ info "COLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
496
+ next
497
+ else
498
+ raise e
499
+ end
500
+ end
501
+ end
502
+
503
+ if Time.now.to_f > start.to_f + timeout
504
+ debug "MISSTIMEOUT PTYPE #{payload_type} #{temperature(payload_type)}"
505
+ raise Skynet::RequestExpiredError.new
506
+ else
507
+ sleepy = rand(sleep_time * 0.5 )
508
+ debug "EMPTY QUEUE #{temperature(payload_type)} SLEEPING: #{sleep_time} / #{sleepy}"
509
+ sleep sleepy
510
+ return false
511
+ end
512
+ end
513
+
514
+ # Skynet::CONFIG[:temperature_growth_rate] ||= 2
515
+ # Skynet::CONFIG[:temperature_backoff_rate] ||= 0.75
516
+
517
+ # TUNEABLE_SETTINGS = [:temp_pow, :temp_interval, :sleep_time]
518
+ #
519
+ # def write_score(new_values,new_result,score)
520
+ # values ||= {}
521
+ # set = new_values.keys.sort.collect{|k|[k,new_values[k]]}.join(",")
522
+ # if not values[set]
523
+ # values[set] ||= {}
524
+ # values[set][:results] ||= []
525
+ # values[set][:scores] ||= []
526
+ # values[set][:settings] ||= {}
527
+ # values[set][:total_score] = 0
528
+ # TUNEABLE_SETTINGS.each do |setting|
529
+ # values[set][:settings][setting] = []
530
+ # end
531
+ # end
532
+ # TUNEABLE_SETTINGS.each do |setting, value|
533
+ # values[set][:settings][setting] << value
534
+ # end
535
+ # values[set][:results] << new_result
536
+ # values[set][:scores] << score + values[set][:total_score]
537
+ # values[set][:total_score] += score
538
+ # end
539
+
540
+ def temperature(payload_type)
541
+ @@temperature[payload_type.to_sym]
542
+ end
543
+
544
+ ## try SQRT *2
545
+ ## try POW 0.6 or .75
546
+ def set_temperature(payload_type,conditions)
547
+ temp_q_conditions = "type = '#{payload_type}' AND updated_on < '#{(Time.now - 5).strftime('%Y-%m-%d %H:%M:%S')}'"
548
+ # "POW(#{(rand(40) + 40) * 0.01})"
549
+ # its almost like the temperature table needs to store the POW and adjust that to be adaptive. Like some % of the time it
550
+ # uses the one in the table, and some % it tries a new one and scores it.
551
+ temperature = SkynetWorkerQueue.connection.select_value(%{select (
552
+ CASE WHEN (@t:=FLOOR(
553
+ POW(@c:=(SELECT count(*) FROM skynet_message_queues WHERE #{conditions}
554
+ ),0.6))) < 1 THEN 1 ELSE @t END) from skynet_queue_temperature WHERE #{temp_q_conditions}
555
+ })
556
+ if temperature
557
+ update("UPDATE skynet_queue_temperature SET temperature = #{temperature} WHERE #{temp_q_conditions}")
558
+ @@temperature[payload_type.to_sym] = temperature.to_f
559
+ else
560
+ sleepy = rand Skynet::CONFIG[:MYSQL_TEMPERATURE_CHANGE_SLEEP]
561
+ sleep sleepy
562
+ @@temperature[payload_type.to_sym] = SkynetWorkerQueue.connection.select_value("select temperature from skynet_queue_temperature WHERE type = '#{payload_type}'").to_f
563
+ end
564
+ # update("UPDATE skynet_queue_temperature SET type = '#{payload_type}', temperature = CASE WHEN @t:=FLOOR(SQRT(select count(*) from skynet_message_queues WHERE #{conditions})) < 1 THEN 1 ELSE @t END")
565
+ # tasks = SkynetMessageQueue.connection.select_value("select count(*) from skynet_message_queues WHERE #{conditions}").to_i
566
+ # sleep 4 if payload_type == :tasks and tasks < 100
567
+ # @@temperature[payload_type.to_sym] = tasks ** 0.5
568
+ # @@temperature[payload_type.to_sym] *= multiplier
569
+ @@temperature[payload_type.to_sym] = 1 if @@temperature[payload_type.to_sym] < 1
570
+ end
571
+ end
572
+ end
573
+ end