skynet 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +4 -0
- data/License.txt +20 -0
- data/Manifest.txt +65 -0
- data/README.txt +100 -0
- data/Rakefile +4 -0
- data/app_generators/skynet_install/USAGE +5 -0
- data/app_generators/skynet_install/skynet_install_generator.rb +84 -0
- data/app_generators/skynet_install/templates/migration.rb +60 -0
- data/app_generators/skynet_install/templates/skynet +33 -0
- data/app_generators/skynet_install/templates/skynet_console +16 -0
- data/bin/skynet +20 -0
- data/bin/skynet_console +9 -0
- data/bin/skynet_install +12 -0
- data/bin/skynet_tuplespace_server +53 -0
- data/config/hoe.rb +74 -0
- data/config/requirements.rb +17 -0
- data/lib/skynet.rb +34 -0
- data/lib/skynet/mapreduce_test.rb +25 -0
- data/lib/skynet/message_queue_adapters/message_queue_adapter.rb +70 -0
- data/lib/skynet/message_queue_adapters/mysql.rb +573 -0
- data/lib/skynet/message_queue_adapters/tuple_space.rb +327 -0
- data/lib/skynet/skynet_active_record_extensions.rb +237 -0
- data/lib/skynet/skynet_config.rb +59 -0
- data/lib/skynet/skynet_console.rb +34 -0
- data/lib/skynet/skynet_console_helper.rb +59 -0
- data/lib/skynet/skynet_debugger.rb +84 -0
- data/lib/skynet/skynet_guid_generator.rb +68 -0
- data/lib/skynet/skynet_job.rb +607 -0
- data/lib/skynet/skynet_launcher.rb +10 -0
- data/lib/skynet/skynet_logger.rb +52 -0
- data/lib/skynet/skynet_manager.rb +486 -0
- data/lib/skynet/skynet_message.rb +366 -0
- data/lib/skynet/skynet_message_queue.rb +100 -0
- data/lib/skynet/skynet_ruby_extensions.rb +36 -0
- data/lib/skynet/skynet_task.rb +76 -0
- data/lib/skynet/skynet_tuplespace_server.rb +82 -0
- data/lib/skynet/skynet_worker.rb +395 -0
- data/lib/skynet/version.rb +9 -0
- data/log/debug.log +0 -0
- data/log/skynet.log +29 -0
- data/log/skynet_tuplespace_server.log +7 -0
- data/log/skynet_worker.pid +1 -0
- data/script/destroy +14 -0
- data/script/generate +14 -0
- data/script/txt2html +74 -0
- data/setup.rb +1585 -0
- data/sometest.rb +23 -0
- data/tasks/deployment.rake +34 -0
- data/tasks/environment.rake +7 -0
- data/tasks/website.rake +17 -0
- data/test/all_models_test.rb +139 -0
- data/test/mysql_message_queue_adaptor_test.rb +199 -0
- data/test/skynet_manager_test.rb +107 -0
- data/test/skynet_message_test.rb +42 -0
- data/test/test_generator_helper.rb +20 -0
- data/test/test_helper.rb +2 -0
- data/test/test_skynet.rb +11 -0
- data/test/test_skynet_install_generator.rb +53 -0
- data/test/tuplespace_message_queue_test.rb +179 -0
- data/tmtags +1242 -0
- data/website/index.html +93 -0
- data/website/index.txt +39 -0
- data/website/javascripts/rounded_corners_lite.inc.js +285 -0
- data/website/stylesheets/screen.css +138 -0
- data/website/template.rhtml +48 -0
- metadata +129 -0
@@ -0,0 +1,53 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'daemons'
|
5
|
+
require 'pp'
|
6
|
+
|
7
|
+
require File.expand_path(File.dirname(__FILE__)) + '/../lib/skynet.rb'
|
8
|
+
|
9
|
+
options = {
|
10
|
+
:port => 7647,
|
11
|
+
:logfile => Skynet::CONFIG[:SKYNET_LOG_FILE],
|
12
|
+
:loglevel => "DEBUG",
|
13
|
+
:piddir => Skynet::CONFIG[:SKYNET_PID_DIR]
|
14
|
+
}
|
15
|
+
|
16
|
+
OptionParser.new do |opt|
|
17
|
+
opt.banner = "Usage: skynet_tuplespace_server (start|stop|run) [options]"
|
18
|
+
opt.on('-t', '--ontop TRUE', 'Dont Daemonize') do |v|
|
19
|
+
options[:ontop] = true if v.upcase == "TRUE" or v == "1"
|
20
|
+
end
|
21
|
+
opt.on('-p', '--port PORT', 'Port to listen on. default 7647') do |v|
|
22
|
+
options[:port] = v.to_i
|
23
|
+
end
|
24
|
+
opt.on('-o', '--log LOGFILE', 'Logfile to log to') do |v|
|
25
|
+
options[:logfile] = v
|
26
|
+
end
|
27
|
+
opt.on('-l', '--loglevel LOGLEVEL', 'Log level defaults to DEBUG') do |v|
|
28
|
+
options[:loglevel] = v
|
29
|
+
end
|
30
|
+
opt.on('-d', '--piddir PIDDIR', 'Directory to put pidfile') do |v|
|
31
|
+
options[:piddir] = File.expand_path(v)
|
32
|
+
end
|
33
|
+
opt.on('-u', '--drburi Drb URI', 'What DRbURI to use') do |v|
|
34
|
+
if v =~ %r{druby://}
|
35
|
+
options[:drburi] = v
|
36
|
+
else
|
37
|
+
options[:drburi] = "druby://#{v}"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
opt.parse!(ARGV)
|
42
|
+
end
|
43
|
+
|
44
|
+
Daemons.run_proc("skynet_tuplespace_server#{options[:port]}",
|
45
|
+
{
|
46
|
+
:dir_mode => :normal,
|
47
|
+
:dir => options[:piddir],
|
48
|
+
:backtrace => true,
|
49
|
+
# :monitor => true,
|
50
|
+
:ontop => options[:ontop] || false
|
51
|
+
}) do
|
52
|
+
server = Skynet::Server.new(options)
|
53
|
+
end
|
data/config/hoe.rb
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'skynet/version'
|
2
|
+
|
3
|
+
AUTHOR = 'Adam Pisoni' # can also be an array of Authors
|
4
|
+
EMAIL = "apisoni@geni.com"
|
5
|
+
DESCRIPTION = "Skynet - A Ruby Map/Reduce Framework"
|
6
|
+
GEM_NAME = 'skynet' # what ppl will type to install your gem
|
7
|
+
RUBYFORGE_PROJECT = 'skynet' # The unix name for your project
|
8
|
+
HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
|
9
|
+
DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
|
10
|
+
|
11
|
+
@config_file = "~/.rubyforge/user-config.yml"
|
12
|
+
@config = nil
|
13
|
+
RUBYFORGE_USERNAME = "unknown"
|
14
|
+
def rubyforge_username
|
15
|
+
unless @config
|
16
|
+
begin
|
17
|
+
@config = YAML.load(File.read(File.expand_path(@config_file)))
|
18
|
+
rescue
|
19
|
+
puts <<-EOS
|
20
|
+
ERROR: No rubyforge config file found: #{@config_file}
|
21
|
+
Run 'rubyforge setup' to prepare your env for access to Rubyforge
|
22
|
+
- See http://newgem.rubyforge.org/rubyforge.html for more details
|
23
|
+
EOS
|
24
|
+
exit
|
25
|
+
end
|
26
|
+
end
|
27
|
+
RUBYFORGE_USERNAME.replace @config["username"]
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
REV = nil
|
32
|
+
# UNCOMMENT IF REQUIRED:
|
33
|
+
# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
|
34
|
+
VERS = Skynet::VERSION::STRING + (REV ? ".#{REV}" : "")
|
35
|
+
RDOC_OPTS = ['--quiet', '--title', 'skynet documentation',
|
36
|
+
"--opname", "index.html",
|
37
|
+
"--line-numbers",
|
38
|
+
"--main", "README",
|
39
|
+
"--inline-source"]
|
40
|
+
|
41
|
+
class Hoe
|
42
|
+
def extra_deps
|
43
|
+
@extra_deps.reject! { |x| Array(x).first == 'hoe' }
|
44
|
+
@extra_deps
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Generate all the Rake tasks
|
49
|
+
# Run 'rake -T' to see list of generated tasks (from gem root directory)
|
50
|
+
hoe = Hoe.new(GEM_NAME, VERS) do |p|
|
51
|
+
p.author = AUTHOR
|
52
|
+
p.description = DESCRIPTION
|
53
|
+
p.email = EMAIL
|
54
|
+
p.summary = DESCRIPTION
|
55
|
+
p.url = HOMEPATH
|
56
|
+
p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
|
57
|
+
p.test_globs = ["test/**/test_*.rb"]
|
58
|
+
p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store'] #An array of file patterns to delete on clean.
|
59
|
+
|
60
|
+
# == Optional
|
61
|
+
p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
|
62
|
+
p.extra_deps = [
|
63
|
+
['daemons',">= 1"],
|
64
|
+
]
|
65
|
+
# An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
|
66
|
+
|
67
|
+
#p.spec_extras = {} # A hash of extra values to set in the gemspec.
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
|
72
|
+
PATH = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
|
73
|
+
hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
|
74
|
+
hoe.rsync_args = '-av --delete --ignore-errors'
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
include FileUtils
|
3
|
+
|
4
|
+
require 'rubygems'
|
5
|
+
%w[rake hoe newgem rubigen].each do |req_gem|
|
6
|
+
begin
|
7
|
+
require req_gem
|
8
|
+
rescue LoadError
|
9
|
+
puts "This Rakefile requires the '#{req_gem}' RubyGem."
|
10
|
+
puts "Installation: gem install #{req_gem} -y"
|
11
|
+
exit
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
|
16
|
+
|
17
|
+
require 'skynet'
|
data/lib/skynet.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)
|
2
|
+
$:.unshift File.dirname(__FILE__) + '/skynet'
|
3
|
+
|
4
|
+
# path = File.expand_path(File.dirname(__FILE__))
|
5
|
+
|
6
|
+
|
7
|
+
require 'drb'
|
8
|
+
require 'skynet_guid_generator'
|
9
|
+
require 'skynet_logger'
|
10
|
+
require 'skynet_config'
|
11
|
+
|
12
|
+
Skynet::CONFIG[:SKYNET_PATH] ||= File.expand_path(File.dirname(__FILE__) +"/..")
|
13
|
+
# Skynet::CONFIG[:LAUNCHER_PATH] ||= File.expand_path(ENV['_'])
|
14
|
+
|
15
|
+
require 'skynet_debugger'
|
16
|
+
require 'skynet_message'
|
17
|
+
require 'message_queue_adapters/message_queue_adapter'
|
18
|
+
require 'message_queue_adapters/tuple_space'
|
19
|
+
require "skynet_message_queue"
|
20
|
+
require 'skynet_job'
|
21
|
+
require 'skynet_worker'
|
22
|
+
require 'skynet_task'
|
23
|
+
require 'skynet_manager'
|
24
|
+
require 'skynet_tuplespace_server'
|
25
|
+
require 'skynet_ruby_extensions'
|
26
|
+
begin
|
27
|
+
require 'active_record'
|
28
|
+
require 'skynet_active_record_extensions'
|
29
|
+
require 'message_queue_adapters/mysql'
|
30
|
+
rescue LoadError => e
|
31
|
+
end
|
32
|
+
require 'mapreduce_test'
|
33
|
+
require 'skynet_launcher'
|
34
|
+
require 'skynet_console'
|
@@ -0,0 +1,25 @@
|
|
1
|
+
class Skynet
|
2
|
+
class MapreduceTest
|
3
|
+
include SkynetDebugger
|
4
|
+
|
5
|
+
def self.map(datas)
|
6
|
+
results = {}
|
7
|
+
datas.each do |data|
|
8
|
+
results[data] ||= 0
|
9
|
+
results[data] += 1
|
10
|
+
end
|
11
|
+
[results]
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.reduce(datas)
|
15
|
+
results = {}
|
16
|
+
datas.each do |hashes|
|
17
|
+
hashes.each do |key,value|
|
18
|
+
results[key] ||= 0
|
19
|
+
results[key] += value
|
20
|
+
end
|
21
|
+
end
|
22
|
+
results
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
class Skynet
|
2
|
+
|
3
|
+
class Error < StandardError
|
4
|
+
end
|
5
|
+
|
6
|
+
class RequestExpiredError < Skynet::Error
|
7
|
+
end
|
8
|
+
|
9
|
+
class InvalidMessage < Skynet::Error
|
10
|
+
end
|
11
|
+
|
12
|
+
class AbstractClassError < Skynet::Error
|
13
|
+
end
|
14
|
+
|
15
|
+
class MessageQueueAdapter
|
16
|
+
|
17
|
+
def list_results(data,timeout=nil)
|
18
|
+
raise AbstractClassError.new("You must implement list_results in a subclass.")
|
19
|
+
end
|
20
|
+
|
21
|
+
def list_tasks(template,timeout=nil)
|
22
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
23
|
+
end
|
24
|
+
|
25
|
+
def take_next_task(template,timeout=nil)
|
26
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
27
|
+
end
|
28
|
+
|
29
|
+
def write_message(template,timeout=nil)
|
30
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
31
|
+
end
|
32
|
+
|
33
|
+
def write_result(template,timeout=nil)
|
34
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
35
|
+
end
|
36
|
+
|
37
|
+
def take_result(template,timeout=nil)
|
38
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
39
|
+
end
|
40
|
+
|
41
|
+
def write_error(template,timeout=nil)
|
42
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
43
|
+
end
|
44
|
+
|
45
|
+
def write_worker_status(template,timeout=nil)
|
46
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
47
|
+
end
|
48
|
+
|
49
|
+
def take_worker_status(template,timeout=nil)
|
50
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
51
|
+
end
|
52
|
+
|
53
|
+
def read_all_worker_statuses(template,timeout=nil)
|
54
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
55
|
+
end
|
56
|
+
|
57
|
+
def get_worker_version(template,timeout=nil)
|
58
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
59
|
+
end
|
60
|
+
|
61
|
+
def set_worker_version(template,timeout=nil)
|
62
|
+
raise AbstractClassError.new("You must implement method in a subclass.")
|
63
|
+
end
|
64
|
+
|
65
|
+
def clear_outstanding_tasks
|
66
|
+
raise AbstractClassError.new("You must implement clear_outstanding_tasks in a subclass.")
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,573 @@
|
|
1
|
+
class SkynetMessageQueue < ActiveRecord::Base
|
2
|
+
end
|
3
|
+
|
4
|
+
class SkynetWorkerQueue < ActiveRecord::Base
|
5
|
+
end
|
6
|
+
|
7
|
+
|
8
|
+
class Skynet
|
9
|
+
|
10
|
+
# require 'mysql'
|
11
|
+
|
12
|
+
class Error < StandardError
|
13
|
+
end
|
14
|
+
|
15
|
+
class RequestExpiredError < Skynet::Error
|
16
|
+
end
|
17
|
+
|
18
|
+
class InvalidMessage < Skynet::Error
|
19
|
+
end
|
20
|
+
|
21
|
+
class MessageQueueAdapter
|
22
|
+
|
23
|
+
class Mysql < Skynet::MessageQueueAdapter
|
24
|
+
|
25
|
+
include SkynetDebugger
|
26
|
+
include Skynet::GuidGenerator
|
27
|
+
|
28
|
+
SEARCH_FIELDS = [:tasktype, :task_id, :job_id, :payload_type, :expire_time, :iteration, :version] unless defined?(SEARCH_FIELDS)
|
29
|
+
|
30
|
+
Skynet::CONFIG[:MYSQL_MESSAGE_QUEUE_TEMP_CHECK_DELAY] ||= 30
|
31
|
+
|
32
|
+
|
33
|
+
@@db_set = false
|
34
|
+
|
35
|
+
def self.adapter
|
36
|
+
:mysql
|
37
|
+
end
|
38
|
+
|
39
|
+
def initialize
|
40
|
+
if Skynet::CONFIG[:QUEUE_DATABASE] and not @@db_set
|
41
|
+
begin
|
42
|
+
SkynetMessageQueue.establish_connection Skynet::CONFIG[:QUEUE_DATABASE]
|
43
|
+
SkynetWorkerQueue.establish_connection Skynet::CONFIG[:QUEUE_DATABASE]
|
44
|
+
rescue ActiveRecord::AdapterNotSpecified => e
|
45
|
+
warn "#{Skynet::CONFIG[:QUEUE_DATABASE]} not defined as a database adaptor #{e.message}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
@@db_set = true
|
49
|
+
end
|
50
|
+
|
51
|
+
# def initialize
|
52
|
+
# SkynetMessageQueue.connection.execute("set session TRANSACTION ISOLATION LEVEL READ UNCOMMITTED")
|
53
|
+
# end
|
54
|
+
|
55
|
+
def self.debug_class_desc
|
56
|
+
"MYSQLMQ"
|
57
|
+
end
|
58
|
+
|
59
|
+
def message_to_conditions(message)
|
60
|
+
template_to_conditions(message.to_a)
|
61
|
+
end
|
62
|
+
|
63
|
+
def template_to_conditions(template,fields=Skynet::Message.fields)
|
64
|
+
fields = fields.invert
|
65
|
+
conditions = []
|
66
|
+
values = []
|
67
|
+
|
68
|
+
fields.keys.each do |field|
|
69
|
+
value = template[fields[field]]
|
70
|
+
next unless value
|
71
|
+
if value.is_a?(Range)
|
72
|
+
conditions << "#{field} BETWEEN #{value.first} AND #{value.last}"
|
73
|
+
elsif value.is_a?(Symbol) or value.is_a?(String)
|
74
|
+
conditions << "#{field} = '#{value}'"
|
75
|
+
else
|
76
|
+
conditions << "#{field} = #{value}"
|
77
|
+
end
|
78
|
+
end
|
79
|
+
return '' if conditions.empty?
|
80
|
+
return conditions.join(" AND ")
|
81
|
+
end
|
82
|
+
|
83
|
+
def message_to_hash(message,timeout=nil,fields=Skynet::Message.fields)
|
84
|
+
hash = {}
|
85
|
+
fields.values.each do |field|
|
86
|
+
next if field == :drburi
|
87
|
+
# next unless message.send(field)
|
88
|
+
if message.send(field).is_a?(Symbol)
|
89
|
+
hash[field] = message.send(field).to_s
|
90
|
+
elsif field == :payload
|
91
|
+
hash[:raw_payload] = message.raw_payload
|
92
|
+
else
|
93
|
+
hash[field] = message.send(field)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
if timeout
|
97
|
+
hash[:timeout] = timeout
|
98
|
+
hash[:expire_time] = (Time.now.to_f + timeout) unless hash[:expire_time]
|
99
|
+
end
|
100
|
+
hash
|
101
|
+
end
|
102
|
+
|
103
|
+
def take_next_task(curver,timeout=0,payload_type=nil)
|
104
|
+
timeout = Skynet::CONFIG[:NEXT_TASK_TIMEOUT] if timeout < 1
|
105
|
+
debug "TASK NEXT TASK!!!!!!! timeout: #{timeout}"
|
106
|
+
message = nil
|
107
|
+
start = Time.now
|
108
|
+
rows = nil
|
109
|
+
loop do
|
110
|
+
# debug "start #{Time.now} timeout #{start + timeout}"
|
111
|
+
message_row = take(Skynet::Message.next_task_template(curver,payload_type),start,timeout)
|
112
|
+
next unless message_row
|
113
|
+
|
114
|
+
begin
|
115
|
+
message = Skynet::Message.new(message_row.attributes)
|
116
|
+
rescue Skynet::Message::BadMessage => e
|
117
|
+
message = nil
|
118
|
+
message_row.destroy
|
119
|
+
next
|
120
|
+
end
|
121
|
+
ftm = message.fallback_task_message
|
122
|
+
rows = update("update skynet_message_queues set iteration = #{ftm.iteration }, expire_time = #{ftm.expire_time} where iteration = #{message.iteration} and id = #{message_row.id}")
|
123
|
+
|
124
|
+
# message = nil if rows == 0
|
125
|
+
return message if message
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
|
130
|
+
|
131
|
+
def write_message(message,timeout=nil)
|
132
|
+
SkynetMessageQueue.create(message_to_hash(message, timeout))
|
133
|
+
end
|
134
|
+
|
135
|
+
def write_result(message,result=[],timeout=nil)
|
136
|
+
result_message = message.result_message(result)
|
137
|
+
result_message.expire_time = nil
|
138
|
+
update_message(result_message,timeout)
|
139
|
+
end
|
140
|
+
|
141
|
+
def update_message(message,timeout=nil)
|
142
|
+
timeout_sql = (timeout ? ", timeout = #{timeout}, expire_time = #{Time.now.to_f + timeout}" : '')
|
143
|
+
rows = 0
|
144
|
+
rows = update(%{
|
145
|
+
update skynet_message_queues
|
146
|
+
set tasktype = "#{message.tasktype}",
|
147
|
+
raw_payload = '#{message.raw_payload}',
|
148
|
+
payload_type = "#{message.payload_type}",
|
149
|
+
tran_id = NULL
|
150
|
+
#{timeout_sql}
|
151
|
+
where task_id = #{message.task_id}
|
152
|
+
})
|
153
|
+
|
154
|
+
raise Skynet::RequestExpiredError.new() if rows == 0
|
155
|
+
end
|
156
|
+
|
157
|
+
def take_result(job_id,timeout=1)
|
158
|
+
start = Time.now
|
159
|
+
result = nil
|
160
|
+
sleep_time = 10
|
161
|
+
if timeout < 1
|
162
|
+
sleep_time = 1
|
163
|
+
elsif timeout > 10
|
164
|
+
sleep_time = 10
|
165
|
+
else
|
166
|
+
sleep_time = timeout * 0.25
|
167
|
+
end
|
168
|
+
message_row = nil
|
169
|
+
|
170
|
+
loop do
|
171
|
+
# message_row = take(Skynet::Message.result_template(job_id), start, timeout,sleep_time)
|
172
|
+
conditions = template_to_conditions(Skynet::Message.result_template(job_id))
|
173
|
+
# sleep_time ||= timeout
|
174
|
+
|
175
|
+
message_row = SkynetMessageQueue.find(:first,:conditions => conditions)
|
176
|
+
break if message_row
|
177
|
+
|
178
|
+
if Time.now.to_f > start.to_f + timeout
|
179
|
+
raise Skynet::RequestExpiredError.new
|
180
|
+
else
|
181
|
+
sleepy = rand(sleep_time)
|
182
|
+
# error "RESULT EMPTY SLEEPING: #{sleepy}"
|
183
|
+
sleep sleepy
|
184
|
+
next
|
185
|
+
end
|
186
|
+
next
|
187
|
+
end
|
188
|
+
|
189
|
+
result = Skynet::Message.new(message_row.clone.attributes)
|
190
|
+
message_row.destroy
|
191
|
+
return result if result
|
192
|
+
end
|
193
|
+
|
194
|
+
def list_tasks(iteration=nil)
|
195
|
+
conditions = template_to_conditions(Skynet::Message.outstanding_tasks_template(iteration))
|
196
|
+
SkynetMessageQueue.find(:all,:conditions => conditions)
|
197
|
+
end
|
198
|
+
|
199
|
+
def list_results
|
200
|
+
conditions = template_to_conditions(Skynet::Message.outstanding_results_template)
|
201
|
+
SkynetMessageQueue.find(:all,:conditions => conditions)
|
202
|
+
end
|
203
|
+
|
204
|
+
def write_error(message,error='',timeout=nil)
|
205
|
+
message.expire_time = nil
|
206
|
+
update_message(message.error_message(error),timeout)
|
207
|
+
end
|
208
|
+
|
209
|
+
def write_worker_status(task, timeout=nil)
|
210
|
+
message = Skynet::WorkerStatusMessage.new(task)
|
211
|
+
worker_fields = Skynet::WorkerStatusMessage.fields.reject {|k,f| f == :process_id or f == :hostname or f == :tasksubtype or f == :tasktype}
|
212
|
+
update_hash = message_to_hash(message, timeout, Skynet::WorkerStatusMessage.fields)
|
213
|
+
update_hash.each do |k,v|
|
214
|
+
if not v
|
215
|
+
update_hash[k] = "NULL"
|
216
|
+
elsif v.kind_of?(String) or v.kind_of?(Symbol)
|
217
|
+
update_hash[k] = "'#{v}'"
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
update_sql = "UPDATE skynet_worker_queues SET #{update_hash.collect{|k,v| "#{k}=#{v}"}.join(',')} WHERE worker_id=#{message.worker_id}"
|
222
|
+
rows = update(update_sql)
|
223
|
+
if rows == 0
|
224
|
+
begin
|
225
|
+
insert_sql = "INSERT INTO skynet_worker_queues (#{update_hash.keys.join(',')}) VALUES (#{update_hash.values.join(',')})"
|
226
|
+
rows = update(insert_sql)
|
227
|
+
rescue ActiveRecord::StatementInvalid => e
|
228
|
+
if e.message =~ /Duplicate/
|
229
|
+
error "DUPLICATE WORKER #{e.message}"
|
230
|
+
else
|
231
|
+
raise e
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
return rows
|
236
|
+
end
|
237
|
+
|
238
|
+
def take_worker_status(task, timeout=nil)
|
239
|
+
conditions = template_to_conditions(Skynet::WorkerStatusMessage.worker_status_template(task), Skynet::WorkerStatusMessage.fields)
|
240
|
+
worker_status = nil
|
241
|
+
SkynetWorkerQueue.transaction do
|
242
|
+
worker_row = SkynetWorkerQueue.find(:first, :conditions => conditions)
|
243
|
+
return unless worker_row
|
244
|
+
worker_status = Skynet::WorkerStatusMessage.new(worker_row.clone.attributes)
|
245
|
+
worker_row.destroy
|
246
|
+
end
|
247
|
+
worker_status
|
248
|
+
end
|
249
|
+
|
250
|
+
def read_all_worker_statuses(hostname=nil,process_id=nil)
|
251
|
+
ws = Skynet::WorkerStatusMessage.all_workers_template(hostname)
|
252
|
+
ws[4] = process_id if process_id
|
253
|
+
conditions = template_to_conditions(ws,Skynet::WorkerStatusMessage.fields)
|
254
|
+
rows = SkynetWorkerQueue.find(:all, :conditions => conditions)
|
255
|
+
workers = rows.collect{ |w| Skynet::WorkerStatusMessage.new(w.attributes) }#.sort{ |a,b| a.process_id <=> b.process_id }
|
256
|
+
end
|
257
|
+
|
258
|
+
|
259
|
+
def clear_worker_status(hostname=nil)
|
260
|
+
if hostname
|
261
|
+
SkynetWorkerQueue.connection.execute("delete from skynet_worker_queues where hostname = '#{hostname}'")
|
262
|
+
else
|
263
|
+
SkynetWorkerQueue.destroy_all
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
def set_worker_version(ver=nil)
|
268
|
+
ver ||= 1
|
269
|
+
# SkynetWorkerQueue.transaction do
|
270
|
+
# SkynetWorkerQueue.connection.execute("delete from skynet_worker_queues where tasktype = 'workerversion'")
|
271
|
+
SkynetWorkerQueue.connection.insert("replace skynet_worker_queues (worker_id, tasktype, version) values (0, 'workerversion',#{ver})")
|
272
|
+
# end
|
273
|
+
ver
|
274
|
+
end
|
275
|
+
|
276
|
+
def get_worker_version
|
277
|
+
# ver = SkynetWorkerQueue.connection.select_value("select min(version) from skynet_message_queues where tasktype = 'task'")
|
278
|
+
ver = SkynetWorkerQueue.connection.select_value("select version from skynet_worker_queues where tasktype = 'workerversion'")
|
279
|
+
if not ver
|
280
|
+
set_worker_version(1)
|
281
|
+
ver = 1
|
282
|
+
end
|
283
|
+
ver.to_i
|
284
|
+
end
|
285
|
+
|
286
|
+
|
287
|
+
def clear_outstanding_tasks
|
288
|
+
SkynetMessageQueue.destroy_all
|
289
|
+
end
|
290
|
+
|
291
|
+
def delete_expired_messages
|
292
|
+
SkynetMessageQueue.connection.delete("delete from skynet_message_queues where expire_time BETWEEN 1 AND '#{Time.now.to_f}'")
|
293
|
+
end
|
294
|
+
|
295
|
+
# select hostname, iteration, count(id) as number_of_workers, count(iteration) as iteration, sum(processed) as processed, max(started_at) as most_recent_task_time from skynet_worker_queues where tasksubtype = 'worker' group by hostname, iteration;
|
296
|
+
#
|
297
|
+
# select hostname, count(id) as number_of_workers, sum(processed) as processed, max(started_at) as most_recent_task_time,
|
298
|
+
# CASE iteration WHEN NULL
|
299
|
+
#
|
300
|
+
# from skynet_worker_queues where tasksubtype = 'worker' group by hostname;
|
301
|
+
|
302
|
+
def stats
|
303
|
+
stats = {
|
304
|
+
:servers => {},
|
305
|
+
:results => 0,
|
306
|
+
:taken_tasks => 0,
|
307
|
+
:untaken_tasks => 0,
|
308
|
+
:taken_master_tasks => 0,
|
309
|
+
:taken_task_tasks => 0,
|
310
|
+
:untaken_master_tasks => 0,
|
311
|
+
:untaken_task_tasks => 0,
|
312
|
+
:processed => 0,
|
313
|
+
:number_of_workers => 0,
|
314
|
+
:active_workers => 0,
|
315
|
+
:idle_workers => 0,
|
316
|
+
:hosts => 0,
|
317
|
+
:masters => 0,
|
318
|
+
:taskworkers => 0,
|
319
|
+
:time => Time.now.to_f
|
320
|
+
}
|
321
|
+
|
322
|
+
stat_rows = SkynetWorkerQueue.connection.select_all(%{
|
323
|
+
SELECT tasktype, payload_type, iteration, count(id) as number_of_tasks
|
324
|
+
FROM skynet_message_queues
|
325
|
+
GROUP BY tasktype, payload_type, iteration
|
326
|
+
})
|
327
|
+
# pp stat_rows
|
328
|
+
stat_rows.each do |row|
|
329
|
+
if row["tasktype"] == "result" or row["payload_type"] == "result"
|
330
|
+
stats[:results] += row["number_of_tasks"].to_i
|
331
|
+
elsif row["tasktype"] == "task"
|
332
|
+
type_of_tasks = nil
|
333
|
+
if row["payload_type"] == "master"
|
334
|
+
type_of_tasks = :master_tasks
|
335
|
+
elsif row["payload_type"] == "task"
|
336
|
+
type_of_tasks = :task_tasks
|
337
|
+
end
|
338
|
+
if row["iteration"].to_i > 0
|
339
|
+
stats["taken_#{type_of_tasks}".to_sym] += row["number_of_tasks"].to_i
|
340
|
+
stats[:taken_tasks] += row["number_of_tasks"].to_i
|
341
|
+
else
|
342
|
+
stats["untaken_#{type_of_tasks}".to_sym] += row["number_of_tasks"].to_i
|
343
|
+
stats[:untaken_tasks] += row["number_of_tasks"].to_i
|
344
|
+
end
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
servers = {}
|
349
|
+
|
350
|
+
stat_sql = <<-SQL
|
351
|
+
select hostname, map_or_reduce, count(id) number_of_workers, sum(processed) as processed,
|
352
|
+
max(started_at) as most_recent_task_time, iteration
|
353
|
+
FROM skynet_worker_queues
|
354
|
+
WHERE skynet_worker_queues.tasksubtype = 'worker'
|
355
|
+
SQL
|
356
|
+
|
357
|
+
stat_rows = SkynetWorkerQueue.connection.select_all("#{stat_sql} GROUP BY hostname, map_or_reduce").each do |row|
|
358
|
+
servers[row["hostname"]] ||= {
|
359
|
+
:processed => 0,
|
360
|
+
:hostname => row["hostname"],
|
361
|
+
:number_of_workers => 0,
|
362
|
+
:active_workers => 0,
|
363
|
+
:idle_workers => 0,
|
364
|
+
}
|
365
|
+
|
366
|
+
servers[row["hostname"]][:processed] += row["processed"].to_i
|
367
|
+
servers[row["hostname"]][:number_of_workers] += row["number_of_workers"].to_i
|
368
|
+
servers[row["hostname"]][:active_workers] += 0
|
369
|
+
servers[row["hostname"]][:idle_workers] += row["number_of_workers"].to_i
|
370
|
+
stats[:processed] += row["processed"].to_i
|
371
|
+
stats[:number_of_workers] += row["number_of_workers"].to_i
|
372
|
+
stats[:idle_workers] += row["number_of_workers"].to_i
|
373
|
+
end
|
374
|
+
|
375
|
+
SkynetWorkerQueue.connection.select_all(%{
|
376
|
+
#{stat_sql} AND skynet_worker_queues.iteration IS NOT NULL
|
377
|
+
GROUP BY hostname, map_or_reduce
|
378
|
+
}).each do |row|
|
379
|
+
map_or_reduce = nil
|
380
|
+
if row["map_or_reduce"] == "master"
|
381
|
+
map_or_reduce = :masters
|
382
|
+
else
|
383
|
+
map_or_reduce = :taskworkers
|
384
|
+
end
|
385
|
+
servers[row["hostname"]][:active_workers] += row["number_of_workers"].to_i
|
386
|
+
servers[row["hostname"]][:idle_workers] -= row["number_of_workers"].to_i
|
387
|
+
servers[row["hostname"]][map_or_reduce] ||= 0
|
388
|
+
servers[row["hostname"]][map_or_reduce] += row["number_of_workers"].to_i
|
389
|
+
stats[map_or_reduce] += row["number_of_workers"].to_i
|
390
|
+
stats[:active_workers] += row["number_of_workers"].to_i
|
391
|
+
stats[:idle_workers] -= row["number_of_workers"].to_i
|
392
|
+
end
|
393
|
+
|
394
|
+
stats[:servers] = servers
|
395
|
+
stats[:hosts] = servers.keys.size
|
396
|
+
stats[:time] = Time.now.to_f - stats[:time]
|
397
|
+
stats
|
398
|
+
end
|
399
|
+
|
400
|
+
def processed(sleepy=5,tim=10)
|
401
|
+
last_time = Time.now
|
402
|
+
last_count = Skynet::MessageQueue.new.stats[:processed]
|
403
|
+
tim.times do
|
404
|
+
new_count = Skynet::MessageQueue.new.stats[:processed]
|
405
|
+
new_time = Time.now
|
406
|
+
puts "Processed #{new_count - last_count} in #{new_time - last_time}"
|
407
|
+
last_time = new_time
|
408
|
+
last_count = new_count
|
409
|
+
sleep sleepy
|
410
|
+
end
|
411
|
+
end
|
412
|
+
|
413
|
+
private
|
414
|
+
|
415
|
+
def update(sql)
|
416
|
+
rows = 0
|
417
|
+
3.times do
|
418
|
+
begin
|
419
|
+
rows = SkynetMessageQueue.connection.update(sql)
|
420
|
+
return rows
|
421
|
+
rescue ActiveRecord::StatementInvalid => e
|
422
|
+
if e.message =~ /Deadlock/ or e.message =~ /Transaction/
|
423
|
+
error "#{self.class} update had collision #{e.message}"
|
424
|
+
sleep 0.1
|
425
|
+
next
|
426
|
+
else
|
427
|
+
raise e
|
428
|
+
end
|
429
|
+
end
|
430
|
+
end
|
431
|
+
return rows
|
432
|
+
end
|
433
|
+
|
434
|
+
Skynet::CONFIG[:MYSQL_TEMPERATURE_CHANGE_SLEEP] ||= 40
|
435
|
+
|
436
|
+
@@temperature ||= {}
|
437
|
+
@@temperature[:task] ||= 1
|
438
|
+
@@temperature[:master] ||= 1
|
439
|
+
@@temperature[:any] ||= 1
|
440
|
+
|
441
|
+
def take(template,start=Time.now,timeout=1,sleep_time=nil)
|
442
|
+
conditions = template_to_conditions(template)
|
443
|
+
sleep_time ||= timeout
|
444
|
+
transaction_id = get_unique_id(1)
|
445
|
+
times_tried = 0
|
446
|
+
payload_type = template[Skynet::Message.fields.invert[:payload_type]]
|
447
|
+
payload_type ||= :any
|
448
|
+
payload_type = payload_type.to_sym
|
449
|
+
|
450
|
+
10.times do
|
451
|
+
begin
|
452
|
+
## TEPERATURE
|
453
|
+
temperature_sql = (temperature(payload_type) > 1 ? " AND id % #{temperature(payload_type).ceil} = #{rand(temperature(payload_type)).to_i} " : '')
|
454
|
+
|
455
|
+
### Mqke sure we get the old ones. If we order by on ever select its VERY expensive.
|
456
|
+
order_by = (payload_type != :master and rand(100) < 5) ? "ORDER BY payload_type desc, created_on desc" : ''
|
457
|
+
|
458
|
+
sql = <<-SQL
|
459
|
+
SELECT *
|
460
|
+
FROM skynet_message_queues
|
461
|
+
WHERE #{conditions} #{temperature_sql}
|
462
|
+
#{order_by}
|
463
|
+
LIMIT 1
|
464
|
+
SQL
|
465
|
+
|
466
|
+
message_row = SkynetMessageQueue.find_by_sql(sql).first
|
467
|
+
if message_row
|
468
|
+
update_conditions = "ID = #{message_row.id} and tran_id "
|
469
|
+
if message_row.tran_id
|
470
|
+
update_conditions << "= #{message_row.tran_id}"
|
471
|
+
else
|
472
|
+
update_conditions << 'IS NULL'
|
473
|
+
end
|
474
|
+
rows = SkynetMessageQueue.connection.update(
|
475
|
+
"UPDATE skynet_message_queues set tran_id = #{transaction_id} WHERE #{update_conditions}"
|
476
|
+
)
|
477
|
+
if rows < 1
|
478
|
+
old_temp = temperature(payload_type)
|
479
|
+
set_temperature(payload_type,conditions)
|
480
|
+
info "MISSCOLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
|
481
|
+
next
|
482
|
+
end
|
483
|
+
return message_row
|
484
|
+
else
|
485
|
+
old_temp = temperature(payload_type)
|
486
|
+
set_temperature(payload_type,conditions)
|
487
|
+
info "MISS PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
|
488
|
+
break if temperature(payload_type) == 1 and old_temp == 1
|
489
|
+
next
|
490
|
+
end
|
491
|
+
rescue ActiveRecord::StatementInvalid => e
|
492
|
+
if e.message =~ /Deadlock/
|
493
|
+
old_temp = temperature(payload_type)
|
494
|
+
set_temperature(payload_type,conditions)
|
495
|
+
info "COLLISION PTYPE #{payload_type} OLDTEMP: #{old_temp} NEWTEMP: #{temperature(payload_type)}"
|
496
|
+
next
|
497
|
+
else
|
498
|
+
raise e
|
499
|
+
end
|
500
|
+
end
|
501
|
+
end
|
502
|
+
|
503
|
+
if Time.now.to_f > start.to_f + timeout
|
504
|
+
debug "MISSTIMEOUT PTYPE #{payload_type} #{temperature(payload_type)}"
|
505
|
+
raise Skynet::RequestExpiredError.new
|
506
|
+
else
|
507
|
+
sleepy = rand(sleep_time * 0.5 )
|
508
|
+
debug "EMPTY QUEUE #{temperature(payload_type)} SLEEPING: #{sleep_time} / #{sleepy}"
|
509
|
+
sleep sleepy
|
510
|
+
return false
|
511
|
+
end
|
512
|
+
end
|
513
|
+
|
514
|
+
# Skynet::CONFIG[:temperature_growth_rate] ||= 2
|
515
|
+
# Skynet::CONFIG[:temperature_backoff_rate] ||= 0.75
|
516
|
+
|
517
|
+
# TUNEABLE_SETTINGS = [:temp_pow, :temp_interval, :sleep_time]
|
518
|
+
#
|
519
|
+
# def write_score(new_values,new_result,score)
|
520
|
+
# values ||= {}
|
521
|
+
# set = new_values.keys.sort.collect{|k|[k,new_values[k]]}.join(",")
|
522
|
+
# if not values[set]
|
523
|
+
# values[set] ||= {}
|
524
|
+
# values[set][:results] ||= []
|
525
|
+
# values[set][:scores] ||= []
|
526
|
+
# values[set][:settings] ||= {}
|
527
|
+
# values[set][:total_score] = 0
|
528
|
+
# TUNEABLE_SETTINGS.each do |setting|
|
529
|
+
# values[set][:settings][setting] = []
|
530
|
+
# end
|
531
|
+
# end
|
532
|
+
# TUNEABLE_SETTINGS.each do |setting, value|
|
533
|
+
# values[set][:settings][setting] << value
|
534
|
+
# end
|
535
|
+
# values[set][:results] << new_result
|
536
|
+
# values[set][:scores] << score + values[set][:total_score]
|
537
|
+
# values[set][:total_score] += score
|
538
|
+
# end
|
539
|
+
|
540
|
+
def temperature(payload_type)
|
541
|
+
@@temperature[payload_type.to_sym]
|
542
|
+
end
|
543
|
+
|
544
|
+
## try SQRT *2
|
545
|
+
## try POW 0.6 or .75
|
546
|
+
def set_temperature(payload_type,conditions)
|
547
|
+
temp_q_conditions = "type = '#{payload_type}' AND updated_on < '#{(Time.now - 5).strftime('%Y-%m-%d %H:%M:%S')}'"
|
548
|
+
# "POW(#{(rand(40) + 40) * 0.01})"
|
549
|
+
# its almost like the temperature table needs to store the POW and adjust that to be adaptive. Like some % of the time it
|
550
|
+
# uses the one in the table, and some % it tries a new one and scores it.
|
551
|
+
temperature = SkynetWorkerQueue.connection.select_value(%{select (
|
552
|
+
CASE WHEN (@t:=FLOOR(
|
553
|
+
POW(@c:=(SELECT count(*) FROM skynet_message_queues WHERE #{conditions}
|
554
|
+
),0.6))) < 1 THEN 1 ELSE @t END) from skynet_queue_temperature WHERE #{temp_q_conditions}
|
555
|
+
})
|
556
|
+
if temperature
|
557
|
+
update("UPDATE skynet_queue_temperature SET temperature = #{temperature} WHERE #{temp_q_conditions}")
|
558
|
+
@@temperature[payload_type.to_sym] = temperature.to_f
|
559
|
+
else
|
560
|
+
sleepy = rand Skynet::CONFIG[:MYSQL_TEMPERATURE_CHANGE_SLEEP]
|
561
|
+
sleep sleepy
|
562
|
+
@@temperature[payload_type.to_sym] = SkynetWorkerQueue.connection.select_value("select temperature from skynet_queue_temperature WHERE type = '#{payload_type}'").to_f
|
563
|
+
end
|
564
|
+
# update("UPDATE skynet_queue_temperature SET type = '#{payload_type}', temperature = CASE WHEN @t:=FLOOR(SQRT(select count(*) from skynet_message_queues WHERE #{conditions})) < 1 THEN 1 ELSE @t END")
|
565
|
+
# tasks = SkynetMessageQueue.connection.select_value("select count(*) from skynet_message_queues WHERE #{conditions}").to_i
|
566
|
+
# sleep 4 if payload_type == :tasks and tasks < 100
|
567
|
+
# @@temperature[payload_type.to_sym] = tasks ** 0.5
|
568
|
+
# @@temperature[payload_type.to_sym] *= multiplier
|
569
|
+
@@temperature[payload_type.to_sym] = 1 if @@temperature[payload_type.to_sym] < 1
|
570
|
+
end
|
571
|
+
end
|
572
|
+
end
|
573
|
+
end
|