skynet 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +99 -0
- data/Manifest.txt +10 -9
- data/README.txt +74 -7
- data/app_generators/skynet_install/skynet_install_generator.rb +26 -22
- data/app_generators/skynet_install/templates/migration.rb +11 -5
- data/app_generators/skynet_install/templates/skynet +25 -12
- data/app_generators/skynet_install/templates/skynet_schema.sql +56 -0
- data/bin/skynet +26 -2
- data/bin/skynet_install +24 -0
- data/bin/skynet_tuplespace_server +13 -0
- data/config/hoe.rb +1 -0
- data/lib/skynet.rb +3 -0
- data/lib/skynet/mapreduce_helper.rb +74 -0
- data/lib/skynet/message_queue_adapters/mysql.rb +225 -172
- data/lib/skynet/message_queue_adapters/tuple_space.rb +31 -16
- data/lib/skynet/skynet_active_record_extensions.rb +78 -46
- data/lib/skynet/skynet_config.rb +162 -23
- data/lib/skynet/skynet_console.rb +23 -10
- data/lib/skynet/skynet_console_helper.rb +61 -58
- data/lib/skynet/skynet_job.rb +741 -493
- data/lib/skynet/skynet_launcher.rb +5 -1
- data/lib/skynet/skynet_manager.rb +106 -49
- data/lib/skynet/skynet_message.rb +169 -174
- data/lib/skynet/skynet_message_queue.rb +29 -16
- data/lib/skynet/skynet_partitioners.rb +92 -0
- data/lib/skynet/skynet_ruby_extensions.rb +3 -4
- data/lib/skynet/skynet_task.rb +61 -19
- data/lib/skynet/skynet_tuplespace_server.rb +0 -2
- data/lib/skynet/skynet_worker.rb +73 -51
- data/lib/skynet/version.rb +1 -1
- data/test/test_active_record_extensions.rb +138 -0
- data/test/test_helper.rb +6 -0
- data/test/{mysql_message_queue_adaptor_test.rb → test_mysql_message_queue_adapter.rb} +94 -30
- data/test/test_skynet.rb +11 -11
- data/test/test_skynet_install_generator.rb +0 -4
- data/test/test_skynet_job.rb +717 -0
- data/test/test_skynet_manager.rb +142 -0
- data/test/test_skynet_message.rb +229 -0
- data/test/test_skynet_task.rb +24 -0
- data/test/{tuplespace_message_queue_test.rb → test_tuplespace_message_queue.rb} +25 -30
- data/website/index.html +56 -16
- data/website/index.txt +55 -25
- data/website/template.rhtml +1 -1
- metadata +29 -13
- data/app_generators/skynet_install/templates/skynet_console +0 -16
- data/bin/skynet_console +0 -9
- data/sometest.rb +0 -23
- data/test/all_models_test.rb +0 -139
- data/test/skynet_manager_test.rb +0 -107
- data/test/skynet_message_test.rb +0 -42
- data/tmtags +0 -1242
@@ -4,6 +4,9 @@ class Skynet
|
|
4
4
|
require 'rubygems'
|
5
5
|
require 'optparse'
|
6
6
|
require 'skynet'
|
7
|
+
require "skynet_console_helper"
|
8
|
+
require "irb/completion"
|
9
|
+
require "irb"
|
7
10
|
|
8
11
|
irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
|
9
12
|
|
@@ -13,22 +16,32 @@ class Skynet
|
|
13
16
|
}
|
14
17
|
|
15
18
|
OptionParser.new do |opt|
|
16
|
-
opt.banner = "Usage:
|
19
|
+
opt.banner = "Usage: skynet console [options]"
|
17
20
|
opt.on("--irb=[#{irb}]", 'Invoke a different irb.') { |v| options[:irb] = v }
|
18
21
|
opt.on('-r', '--required LIBRARY', 'Require the specified libraries. To include multiple libraries, include multiple -r options. ie. -r skynet -r fileutils') do |v|
|
19
22
|
options[:required_libs] << File.expand_path(v)
|
20
23
|
end
|
21
24
|
opt.parse!(ARGV)
|
22
25
|
end
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
26
|
+
|
27
|
+
options[:required_libs] + libs
|
28
|
+
options[:required_libs].uniq.each do |lib|
|
29
|
+
require lib
|
30
|
+
end
|
31
|
+
|
32
|
+
IRB.setup(Skynet::CONFIG[:LAUNCHER_PATH])
|
33
|
+
IRB.conf[:PROMPT_MODE] = :SIMPLE
|
34
|
+
irb = IRB::Irb.new()
|
35
|
+
IRB.conf[:MAIN_CONTEXT] = irb.context
|
36
|
+
irb.context.workspace.main.extend Skynet::ConsoleHelper
|
37
|
+
|
38
|
+
trap("SIGINT") do
|
39
|
+
irb.signal_handle
|
40
|
+
end
|
41
|
+
|
42
|
+
catch(:IRB_EXIT) do
|
43
|
+
irb.eval_input
|
44
|
+
end
|
32
45
|
end
|
33
46
|
end
|
34
47
|
end
|
@@ -1,59 +1,62 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
1
|
+
module Skynet::ConsoleHelper
|
2
|
+
# All of these commands can be run at the 'skynet console'.
|
3
|
+
|
4
|
+
def mq
|
5
|
+
@mq ||= Skynet::MessageQueue.new
|
6
|
+
end
|
7
|
+
|
8
|
+
def stats
|
9
|
+
mq.stats
|
10
|
+
end
|
11
|
+
|
12
|
+
def increment_worker_version
|
13
|
+
mq.increment_worker_version
|
14
|
+
end
|
15
|
+
|
16
|
+
def get_worker_version
|
17
|
+
mq.get_worker_version
|
18
|
+
end
|
19
|
+
|
20
|
+
def set_worker_version(*args)
|
21
|
+
mq.set_worker_version(*args)
|
22
|
+
end
|
23
|
+
|
24
|
+
def manager
|
25
|
+
@manager ||= DRbObject.new(nil,Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL])
|
26
|
+
end
|
27
|
+
|
28
|
+
def add_lib(lib)
|
29
|
+
manager.required_libs << File.expand_path(lib)
|
30
|
+
manager.restart_workers
|
31
|
+
end
|
32
|
+
|
33
|
+
def restart_workers
|
34
|
+
manager.restart_workers
|
35
|
+
end
|
36
|
+
|
37
|
+
def add_workers(num)
|
38
|
+
manager.add_workers(num)
|
39
|
+
end
|
40
|
+
|
41
|
+
def remove_workers(num)
|
42
|
+
manager.remove_workers(num)
|
43
|
+
end
|
44
|
+
|
45
|
+
# ===============
|
46
|
+
# = Doesnt work =
|
47
|
+
# ===============
|
48
|
+
# def help
|
49
|
+
# puts <<-HELP
|
50
|
+
# mq
|
51
|
+
# stats
|
52
|
+
# increment_worker_version
|
53
|
+
# get_worker_version
|
54
|
+
# set_worker_version(version)
|
55
|
+
# manager
|
56
|
+
# add_lib(library_to_include) -- forces a restart
|
57
|
+
# restart_workers
|
58
|
+
# add_workers(number_of_workers)
|
59
|
+
# remove_workers(number_of_workers)
|
60
|
+
# HELP
|
61
|
+
# end
|
24
62
|
end
|
25
|
-
|
26
|
-
def add_lib(lib)
|
27
|
-
manager.required_libs << File.expand_path(lib)
|
28
|
-
manager.restart_workers
|
29
|
-
end
|
30
|
-
|
31
|
-
def restart_workers
|
32
|
-
manager.restart_workers
|
33
|
-
end
|
34
|
-
|
35
|
-
def add_workers(num)
|
36
|
-
manager.add_workers(num)
|
37
|
-
end
|
38
|
-
|
39
|
-
def remove_workers(num)
|
40
|
-
manager.remove_workers(num)
|
41
|
-
end
|
42
|
-
|
43
|
-
# ===============
|
44
|
-
# = Doesnt work =
|
45
|
-
# ===============
|
46
|
-
# def help
|
47
|
-
# puts <<-HELP
|
48
|
-
# mq
|
49
|
-
# stats
|
50
|
-
# increment_worker_version
|
51
|
-
# get_worker_version
|
52
|
-
# set_worker_version(version)
|
53
|
-
# manager
|
54
|
-
# add_lib(library_to_include) -- forces a restart
|
55
|
-
# restart_workers
|
56
|
-
# add_workers(number_of_workers)
|
57
|
-
# remove_workers(number_of_workers)
|
58
|
-
# HELP
|
59
|
-
# end
|
data/lib/skynet/skynet_job.rb
CHANGED
@@ -1,607 +1,855 @@
|
|
1
|
-
# require 'ruby2ruby' # XXX this will break unless people have the fix to Ruby2Ruby
|
2
|
-
##### ruby2ruby fix from ruby2ruby.rb ############
|
3
|
-
### XXX This is bad. Some people rely on an exception being thrown if a method is missing! BULLSHIT!
|
4
|
-
# class NilClass # Objective-C trick
|
5
|
-
# def method_missing(msg, *args, &block)
|
6
|
-
# nil
|
7
|
-
# end
|
8
|
-
# end
|
9
|
-
##############################
|
10
|
-
|
11
|
-
# Users should create instances of this class. Rather than subclassing,
|
12
|
-
# jobs are specialized by assigning lambdas to map, reduce, and partition.
|
13
|
-
# This allows the instance to easily create sub-tasks and marshal the map
|
14
|
-
# and reduce code for sending to workers.
|
15
|
-
#
|
16
|
-
|
17
1
|
class Skynet
|
2
|
+
# Skynet::Job is the main interface to Skynet. You create a job object giving
|
3
|
+
# it the starting data (map_data), along with what class has the map/reduce
|
4
|
+
# functions in it. Even though Skynet is distributed, when you call #run on
|
5
|
+
# a plain Skynet::Job, it will still block in your current process until it has completed
|
6
|
+
# your task. If you want to go on to do other things you'll want to pass :async => true
|
7
|
+
# when creating a new job. Then later call job.results to retrieve your results.
|
8
|
+
#
|
9
|
+
# There are also many global configuration options which can be controlled through Skynet::CONFIG
|
10
|
+
#
|
11
|
+
# Example Usage:
|
12
|
+
#
|
13
|
+
# class MapReduceTest
|
14
|
+
#
|
15
|
+
# def self.run
|
16
|
+
# job = Skynet::Job.new(
|
17
|
+
# :mappers => 2,
|
18
|
+
# :reducers => 1,
|
19
|
+
# :map_reduce_class => self,
|
20
|
+
# :map_data => [OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 3})]
|
21
|
+
# )
|
22
|
+
# results = job.run
|
23
|
+
# end
|
24
|
+
#
|
25
|
+
# def self.map(profiles)
|
26
|
+
# result = Array.new
|
27
|
+
# profiles.each do |profile|
|
28
|
+
# result << [profile.created_by, 1] if profile.created_by
|
29
|
+
# end
|
30
|
+
# result
|
31
|
+
# end
|
32
|
+
#
|
33
|
+
# def self.reduce(pairs)
|
34
|
+
# totals = Hash.new
|
35
|
+
# pairs.each do |pair|
|
36
|
+
# created_by, count = pair[0], pair[1]
|
37
|
+
# totals[created_by] ||= 0
|
38
|
+
# totals[created_by] += count
|
39
|
+
# end
|
40
|
+
# totals
|
41
|
+
# end
|
42
|
+
# end
|
43
|
+
#
|
44
|
+
# MapReduceTest.run
|
45
|
+
#
|
46
|
+
# You might notice that self.map and self.reduce both accept Arrays. If you do not want to deal with
|
47
|
+
# getting arrays of map_data or reduce_data, you can include MapreduceHelper into your class and then
|
48
|
+
# implement self.map_each and self.reduce_each methods. The included self.map and self.reduce methods
|
49
|
+
# will handle iterating over the map_data and reduce_data, passing each element to your map_each and
|
50
|
+
# reduce_each methods respectively. They will also handle error handling within that loop to make sure
|
51
|
+
# even if a single map or reduce fails, processing will continue. If you do not want processing to
|
52
|
+
# continue if a map fails, do not use the MapreduceHelper mixin.
|
53
|
+
#
|
54
|
+
# There are many other options to control various defaults and timeouts.
|
18
55
|
class Job
|
19
56
|
include SkynetDebugger
|
20
57
|
include Skynet::GuidGenerator
|
21
58
|
|
22
|
-
class WorkerError < Skynet::Error
|
23
|
-
end
|
59
|
+
class WorkerError < Skynet::Error; end
|
24
60
|
|
25
|
-
class BadMapOrReduceError < Skynet::Error
|
26
|
-
end
|
61
|
+
class BadMapOrReduceError < Skynet::Error; end
|
27
62
|
|
28
|
-
class Error < Skynet::Error
|
29
|
-
end
|
63
|
+
class Error < Skynet::Error; end
|
30
64
|
|
31
|
-
@@svn_rev = nil
|
32
65
|
@@worker_ver = nil
|
33
|
-
@@log = nil
|
34
66
|
|
35
|
-
FIELDS = [:
|
36
|
-
:reduce_timeout, :master_timeout, :
|
37
|
-
:master_result_timeout, :result_timeout, :start_after, :solo, :single,
|
38
|
-
:map, :map_partitioner, :reduce, :
|
67
|
+
FIELDS = [:queue_id, :mappers, :reducers, :silent, :name, :map_timeout, :map_data, :job_id,
|
68
|
+
:reduce_timeout, :master_timeout, :map_name, :reduce_name,
|
69
|
+
:master_result_timeout, :result_timeout, :start_after, :solo, :single, :version,
|
70
|
+
:map, :map_partitioner, :reduce, :reduce_partition, :map_reduce_class,
|
71
|
+
:master_retry, :map_retry, :reduce_retry,
|
72
|
+
:keep_map_tasks, :keep_reduce_tasks,
|
73
|
+
:local_master, :async
|
39
74
|
]
|
40
75
|
|
41
76
|
FIELDS.each do |method|
|
42
|
-
|
43
|
-
|
77
|
+
if [:map_reduce_class, :version, :map, :reduce, :map_data, :start_after].include?(method)
|
78
|
+
attr_reader method
|
79
|
+
elsif [:master_retry, :map_retry, :reduce_retry,:keep_map_tasks, :keep_reduce_tasks].include?(method)
|
80
|
+
attr_writer method
|
81
|
+
else
|
82
|
+
attr_accessor method
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
attr_accessor :use_local_queue
|
87
|
+
|
88
|
+
Skynet::CONFIG[:JOB_DEFAULTS] = {
|
89
|
+
:queue_id => 0,
|
90
|
+
:mappers => 2,
|
91
|
+
:reducers => 1,
|
92
|
+
:map_timeout => 60,
|
93
|
+
:reduce_timeout => 60,
|
94
|
+
:master_timeout => 60,
|
95
|
+
:result_timeout => 1200,
|
96
|
+
:start_after => 0,
|
97
|
+
:master_result_timeout => 1200,
|
98
|
+
:local_master => true
|
99
|
+
}
|
44
100
|
|
45
|
-
|
46
101
|
def self.debug_class_desc
|
47
102
|
"JOB"
|
48
103
|
end
|
49
104
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
105
|
+
# Most of the time you will merely call #new(options) and then #run on the returned object.
|
106
|
+
#
|
107
|
+
# Options are:
|
108
|
+
# <tt>:local_master</tt> BOOL (DEFAULT true)
|
109
|
+
# By default, your Skynet::Job will act as the master for your map/reduce job, doling out
|
110
|
+
# tasks, waiting for other workers to complete and return their results and dealing with
|
111
|
+
# merging and partitioning the data. If you call #run in async mode, another worker will handle
|
112
|
+
# being the master for your job without blocking. If you run :async => false, :local_master => false
|
113
|
+
# Skynet will let another worker be the master for your job, but will block waiting for the
|
114
|
+
# final results. The benefit of this is that if your process dies, the Job will continue to
|
115
|
+
# run remotely.
|
116
|
+
#
|
117
|
+
# <tt>:async</tt> BOOL (DEFAULT false)
|
118
|
+
# If you run in async mode, another worker will handle being the master for your job without blocking.
|
119
|
+
# You can not pass :local_master => true, :async => true since the only way to allow your
|
120
|
+
# job to run asyncronously is to have a remote_master.
|
121
|
+
#
|
122
|
+
# <tt>:map_data</tt>(Array or Enumerable)
|
123
|
+
# map_data should be an Array or Enumerable that data Skynet::Job will split up
|
124
|
+
# and distribute among your workers. You can stream data to Skynet::Job by passing
|
125
|
+
# an Enumerable that implements next or each.
|
126
|
+
#
|
127
|
+
# <tt>:map_reduce_class</tt>(Class or Class Name)
|
128
|
+
# Skynet::Job will look for class methods named self.map, self.reduce, self.map_partitioner,
|
129
|
+
# self.reduce_partition in your map_reduce_class. The only method requires is self.map.
|
130
|
+
# Each of these methods must accept an array. Examples above.
|
131
|
+
#
|
132
|
+
# <tt>:map</tt>(Class Name)
|
133
|
+
# You can pass a classname, or a proc. If you pass a classname, Job will look for a method
|
134
|
+
# called self.map in that class.
|
135
|
+
# WARNING: Passing a proc does not work right now.
|
136
|
+
#
|
137
|
+
# <tt>:reduce</tt>(Class Name)
|
138
|
+
# You can pass a classname, or a proc. If you pass a classname, Job will look for a method
|
139
|
+
# called self.reduce in that class.
|
140
|
+
# WARNING: Passing a proc does not work right now.
|
141
|
+
#
|
142
|
+
# <tt>:reduce_partition</tt>(Class Name)
|
143
|
+
# You can pass a classname, or a proc. If you pass a classname, Job will look for a method
|
144
|
+
# called self.reduce_partition in that class.
|
145
|
+
# WARNING: Passing a proc does not work right now.
|
146
|
+
#
|
147
|
+
# <tt>:mappers</tt> Fixnum
|
148
|
+
# The number of mappers to partition map data for.
|
149
|
+
#
|
150
|
+
# <tt>:reducers</tt> Fixnum
|
151
|
+
# The number of reducers to partition the returned map_data for.
|
152
|
+
#
|
153
|
+
# <tt>:master_retry</tt> Fixnum
|
154
|
+
# If the master fails for any reason, how many times should it be retried? You can also set
|
155
|
+
# Skynet::CONFIG[:DEFAULT_MASTER_RETRY] (DEFAULT 0)
|
156
|
+
#
|
157
|
+
# <tt>:map_retry</tt> Fixnum
|
158
|
+
# If a map task fails for any reason, how many times should it be retried? You can also set
|
159
|
+
# Skynet::CONFIG[:DEFAULT_MAP_RETRY] (DEFAULT 3)
|
160
|
+
#
|
161
|
+
# <tt>:reduce_retry</tt> Fixnum
|
162
|
+
# If a reduce task fails for any reason, how many times should it be retried? You can also set
|
163
|
+
# Skynet::CONFIG[:DEFAULT_REDUCE_RETRY] (DEFAULT 3)
|
164
|
+
#
|
165
|
+
# <tt>:master_timeout</tt>, <tt>:map_timeout</tt>, <tt>:reduce_timeout</tt>, <tt>master_result_timeout</tt>, <tt>result_timeout</tt>
|
166
|
+
# These control how long skynet should wait for particular actions to be finished.
|
167
|
+
# The master_timeout controls how long the master should wait for ALL map/reduce tasks ie. the entire job to finish.
|
168
|
+
# The master_result_timeout controls how long the final result should wait in the queue before being expired.
|
169
|
+
# The map and reduce timeouts control how long individual map and reduce tasks shoudl take.
|
170
|
+
#
|
171
|
+
# <tt>:single</tt> BOOL
|
172
|
+
# By default the master task distributes the map and reduce tasks to other workers.
|
173
|
+
# In single mode the master will take care of the map and reduce tasks by itself.
|
174
|
+
# This is handy when you really want to just perform some single action asyncronously.
|
175
|
+
# In this case you're merely using Skynet to postpone some action. In single mode, the
|
176
|
+
# first worker that picks up your task will just complete it as opposed to trying to distribute
|
177
|
+
# it to another worker.
|
178
|
+
#
|
179
|
+
# <tt>:start_after</tt> Time or Time.to_i
|
180
|
+
# Do not start job until :start_after has passed
|
181
|
+
#
|
182
|
+
# <tt>:queue</tt> String
|
183
|
+
# Which queue should this Job go in to? The queue provided is merely used to
|
184
|
+
# determine the queue_id.
|
185
|
+
# Queues are defined in Skynet::CONFIG[:MESSAGE_QUEUES]
|
186
|
+
#
|
187
|
+
# <tt>:queue_id</tt> Fixnum (DEFAULT 0)
|
188
|
+
# Which queue should this Job go in to?
|
189
|
+
# Queues are defined in Skynet::CONFIG[:MESSAGE_QUEUES]
|
190
|
+
#
|
191
|
+
# <tt>:solo</tt> BOOL
|
192
|
+
# One normally turns solo mode in in Skynet::Config using Skynet::CONFIG[:SOLO] = true
|
193
|
+
# In solo mode, Skynet jobs do not add items to a Skynet queue. Instead they do all
|
194
|
+
# work in place. It's like a Skynet simulation mode. It will complete all tasks
|
195
|
+
# without Skynet running. Great for testing. You can also wrap code blocks in
|
196
|
+
# Skynet.solo {} to run that code in solo mode.
|
197
|
+
#
|
198
|
+
# <tt>:version</tt> Fixnum
|
199
|
+
# If you do not provide a version the current worker version will be used.
|
200
|
+
# Skynet workers start at a specific version and only look for jobs that match that version.
|
201
|
+
# A worker will continue looking for jobs at that version until there are no more jobs left on
|
202
|
+
# the queue for that version. At that time, the worker will check to see if there is a new version.
|
203
|
+
# If there is, it will restart itself at the new version (assuming you had already pushed code to
|
204
|
+
# said workers.)
|
205
|
+
# To retrieve the current version, set the current version or increment the current version, see
|
206
|
+
# Skynet::Job.set_worker_version, Skynet::Job.get_worker_version, Skynet::Job.increment_worker_version
|
207
|
+
#
|
208
|
+
# <tt>:name</tt>, <tt>:map_name</tt>, <tt>:reduce_name</tt>
|
209
|
+
# These name methods are merely for debugging while watching the Skynet logs or the Skynet queue.
|
210
|
+
# If you do not supply names, it will try and provide sensible ones based on your class names.
|
211
|
+
#
|
212
|
+
# <tt>:keep_map_tasks</tt> BOOL or Fixnum (DEFAULT 1)
|
213
|
+
# If true, the master will run the map_tasks locally.
|
214
|
+
# If a number is provided, the master will run the map_tasks locally if there are
|
215
|
+
# LESS THAN OR EQUAL TO the number provided.
|
216
|
+
# You may also set Skynet::CONFIG[:DEFAILT_KEEP_MAP_TASKS] DEFAULT 1
|
217
|
+
#
|
218
|
+
# <tt>:keep_reduce_tasks</tt> BOOL or Fixnum (DEFAULT 1)
|
219
|
+
# If true, the master will run the reduce_tasks locally.
|
220
|
+
# If a number is provided, the master will run the reduce_tasks locally if there are
|
221
|
+
# LESS THAN OR EQUAL TO the number provided.
|
222
|
+
# You may also set Skynet::CONFIG[:DEFAILT_REDUCVE_MAP_TASKS] DEFAULT 1
|
223
|
+
def initialize(options = {})
|
224
|
+
FIELDS.each do |field|
|
225
|
+
if options.has_key?(field)
|
226
|
+
self.send("#{field}=".to_sym,options[field])
|
227
|
+
elsif Skynet::CONFIG[:JOB_DEFAULTS][field]
|
228
|
+
self.send("#{field}=".to_sym,Skynet::CONFIG[:JOB_DEFAULTS][field])
|
229
|
+
end
|
230
|
+
if options[:queue]
|
231
|
+
raise Error.new("The provided queue (#{options[:queue]}) does not exist in Skynet::CONFIG[:MESSAGE_QUEUES]") unless Skynet::CONFIG[:MESSAGE_QUEUES].index(options[:queue])
|
232
|
+
self.queue_id = Skynet::CONFIG[:MESSAGE_QUEUES].index(options[:queue])
|
233
|
+
end
|
234
|
+
|
235
|
+
# Backward compatability
|
236
|
+
self.mappers ||= options[:map_tasks]
|
237
|
+
self.reducers ||= options[:reduce_tasks]
|
238
|
+
end
|
239
|
+
|
240
|
+
raise Error.new("You can not run a local master in async mode.") if self.async and self.local_master
|
76
241
|
|
77
242
|
@job_id = task_id
|
78
243
|
end
|
79
244
|
|
80
|
-
|
81
|
-
|
82
|
-
|
245
|
+
# Options are:
|
246
|
+
# <tt>:local_master</tt> BOOL (DEFAULT true)
|
247
|
+
# By default, your Skynet::Job will act as the master for your map/reduce job, doling out
|
248
|
+
# tasks, waiting for other workers to complete and return their results and dealing with
|
249
|
+
# merging and partitioning the data. If you run in async mode, another worker will handle
|
250
|
+
# being the master for your job without blocking. If you run :async => false, :local_master => false
|
251
|
+
# Skynet will let another worker be the master for your job, but will block waiting for the
|
252
|
+
# final results. The benefit of this is that if your process dies, the Job will continue to
|
253
|
+
# run remotely.
|
254
|
+
#
|
255
|
+
# <tt>:async</tt> BOOL (DEFAULT false)
|
256
|
+
# If you run in async mode, another worker will handle being the master for your job without blocking.
|
257
|
+
# You can not pass :local_master => true, :async => true since the only way to allow your
|
258
|
+
# job to run asyncronously is to have a remote_master.
|
259
|
+
#
|
260
|
+
# You can pass any options you might pass to Skynet::Job.new. Warning: Passing options to run
|
261
|
+
# will permanently change properties of the job.
|
262
|
+
def run(options = {})
|
263
|
+
FIELDS.each do |field|
|
264
|
+
if options.has_key?(field)
|
265
|
+
self.send("#{field}=".to_sym,options[field])
|
266
|
+
end
|
83
267
|
end
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
268
|
+
raise Error.new("You can not run a local master in async mode.") if self.async and self.local_master
|
269
|
+
|
270
|
+
info "RUN 1 BEGIN #{name}, job_id:#{job_id} vers: #{version} async:#{async}, local_master: #{local_master}, master?: #{master?}"
|
271
|
+
|
272
|
+
# run the master task if we're running async or local_master
|
273
|
+
if master?
|
274
|
+
master_enqueue
|
275
|
+
# ====================================================================================
|
276
|
+
# = FIXME If async Return a handle to an object that can used to retrieve the results later.
|
277
|
+
# ====================================================================================
|
278
|
+
async? ? job_id : master_results
|
279
|
+
else
|
280
|
+
number_of_tasks_queued = self.map_enqueue
|
281
|
+
map_results = self.map_results(number_of_tasks_queued)
|
282
|
+
return map_results unless map_results and self.reduce
|
283
|
+
|
284
|
+
partitioned_data = self.partition_data(map_results)
|
285
|
+
return unless partitioned_data
|
286
|
+
number_of_tasks_queued = self.reduce_enqueue(partitioned_data)
|
287
|
+
|
288
|
+
@results = self.reduce_results(number_of_tasks_queued)
|
88
289
|
end
|
89
|
-
hash
|
90
290
|
end
|
91
|
-
|
92
|
-
def
|
93
|
-
|
291
|
+
|
292
|
+
def master_enqueue
|
293
|
+
self.use_local_queue = local_master?
|
294
|
+
messages = tasks_to_messages([master_task])
|
295
|
+
enqueue_messages(messages)
|
94
296
|
end
|
95
297
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
#
|
100
|
-
#
|
101
|
-
#
|
102
|
-
|
103
|
-
# end
|
298
|
+
# Returns the final results of this map/reduce job. If results is called on an :async job
|
299
|
+
# calling results will block until results are found or the master_timeout is reached.
|
300
|
+
def results
|
301
|
+
# ============================================
|
302
|
+
# = FIXME Maybe this can have better warnings if the results aren't ready yet. =
|
303
|
+
# ============================================
|
304
|
+
master_results
|
104
305
|
end
|
105
|
-
|
106
|
-
def
|
107
|
-
|
108
|
-
@@worker_version ||= mq.get_worker_version
|
109
|
-
@version ||= @@worker_version
|
306
|
+
|
307
|
+
def master_results
|
308
|
+
@results = gather_results(1,master_timeout,name) unless defined?(@results)
|
110
309
|
end
|
111
310
|
|
112
|
-
def
|
113
|
-
|
311
|
+
def map_enqueue
|
312
|
+
task_ids = []
|
313
|
+
map_tasks = self.map_tasks
|
314
|
+
self.use_local_queue = map_local?
|
315
|
+
if map_tasks
|
316
|
+
number_of_tasks = 0
|
317
|
+
map_tasks.each do |task|
|
318
|
+
number_of_tasks += 1
|
319
|
+
enqueue_messages(tasks_to_messages(task))
|
320
|
+
end
|
321
|
+
end
|
322
|
+
return number_of_tasks
|
114
323
|
end
|
115
324
|
|
116
|
-
def
|
117
|
-
"#{
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
newver
|
124
|
-
end
|
125
|
-
|
126
|
-
def solo?
|
127
|
-
(@solo or CONFIG[:SOLO])
|
128
|
-
end
|
129
|
-
|
130
|
-
def single?
|
131
|
-
@single
|
325
|
+
def map_results(number_of_tasks)
|
326
|
+
debug "RUN MAP 2.4 BEFORE MAP #{display_info} MAP_LOCAL?:#{map_local?} USE_LOCAL_QUEUE?:#{use_local_queue?}"
|
327
|
+
results = gather_results(number_of_tasks, map_timeout, map_name)
|
328
|
+
return unless results
|
329
|
+
results.compact! if results.is_a?(Array)
|
330
|
+
debug "RUN MAP 2.5 RESULTS AFTER RUN #{display_info} MAP_LOCAL:#{map_local?} USE_LOCAL_QUEUE?:#{use_local_queue?} results:", results.inspect
|
331
|
+
results
|
132
332
|
end
|
133
|
-
|
134
|
-
def
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
task_ids << task.task_id
|
150
|
-
worker_message = Skynet::Message.new(
|
151
|
-
:tasktype => :task,
|
152
|
-
:job_id => job_id,
|
153
|
-
:task_id => task.task_id,
|
154
|
-
:payload => task,
|
155
|
-
:payload_type => task.task_or_master,
|
156
|
-
:expiry => timeout,
|
157
|
-
:expire_time => @start_after,
|
158
|
-
:iteration => 0,
|
159
|
-
:name => description,
|
160
|
-
:version => @version
|
161
|
-
)
|
162
|
-
debug "RUN TASKS WORKER MESSAGE #{description} job_id: #{job_id}", worker_message.to_a
|
163
|
-
mq.write_message(worker_message,timeout * 5)
|
333
|
+
|
334
|
+
def partition_data(post_map_data)
|
335
|
+
debug "RUN REDUCE 3.1 BEFORE PARTITION #{display_info} reducers: #{reducers}"
|
336
|
+
debug "RUN REDUCE 3.1 : #{reducers} #{name}, job_id:#{job_id}", post_map_data
|
337
|
+
return unless post_map_data
|
338
|
+
partitioned_data = nil
|
339
|
+
if not @reduce_partition
|
340
|
+
# =====================
|
341
|
+
# = XXX HACK
|
342
|
+
# = There was a bug in Job where the reduce_partition of master jobs wasn't being set! This is to catch that.
|
343
|
+
# = It handles it by checking if the map class has a reduce partitioner. Maybe this is a good thing to leave anyway.
|
344
|
+
# =====================
|
345
|
+
if @map.is_a?(String) and @map.constantize.respond_to?(:reduce_partition)
|
346
|
+
partitioned_data = @map.constantize.reduce_partition(post_map_data, reducers)
|
347
|
+
else
|
348
|
+
partitioned_data = Skynet::Partitioners::RecombineAndSplit.reduce_partition(post_map_data, reducers)
|
164
349
|
end
|
350
|
+
elsif @reduce_partition.is_a?(String)
|
351
|
+
partitioned_data = @reduce_partition.constantize.reduce_partition(post_map_data, reducers)
|
352
|
+
else
|
353
|
+
partitioned_data = @reduce_partition.call(post_map_data, reducers)
|
165
354
|
end
|
355
|
+
partitioned_data.compact! if partitioned_data
|
356
|
+
debug "RUN REDUCE 3.2 AFTER PARTITION #{display_info} reducers: #{reducers}"
|
357
|
+
debug "RUN REDUCE 3.2 AFTER PARTITION #{display_info} data:", partitioned_data if partitioned_data
|
358
|
+
partitioned_data
|
359
|
+
end
|
166
360
|
|
167
|
-
|
168
|
-
return
|
169
|
-
|
170
|
-
|
361
|
+
def reduce_enqueue(partitioned_data)
|
362
|
+
return partitioned_data unless @reduce and reducers and reducers > 0
|
363
|
+
debug "RUN REDUCE 3.3 CREATED REDUCE TASKS #{display_info}", partitioned_data
|
364
|
+
|
365
|
+
reduce_tasks = self.reduce_tasks(partitioned_data)
|
366
|
+
self.use_local_queue = reduce_local?(reduce_tasks)
|
367
|
+
number_of_tasks = 0
|
368
|
+
reduce_tasks.each do |task|
|
369
|
+
number_of_tasks += 1
|
370
|
+
enqueue_messages(tasks_to_messages(task))
|
371
|
+
end
|
372
|
+
return number_of_tasks
|
373
|
+
end
|
171
374
|
|
172
|
-
|
375
|
+
def reduce_results(number_of_tasks)
|
376
|
+
results = gather_results(number_of_tasks, reduce_timeout, reduce_name)
|
377
|
+
if results.is_a?(Array) and results.first.is_a?(Hash)
|
378
|
+
hash_results = Hash.new
|
379
|
+
results.each {|h| hash_results.merge!(h) if h.class == Hash}
|
380
|
+
results = hash_results
|
381
|
+
elsif results.is_a?(Array) and results.first.is_a?(Array)
|
382
|
+
results = results.compact
|
383
|
+
end
|
384
|
+
debug "RUN REDUCE 3.4 AFTER REDUCE #{display_info} results size: #{results ? results.size : ''}"
|
385
|
+
debug "RUN REDUCE 3.4 AFTER REDUCE #{display_info} results:", results if results
|
386
|
+
return results
|
387
|
+
end
|
388
|
+
|
389
|
+
def enqueue_messages(messages)
|
390
|
+
messages.each do |message|
|
391
|
+
timeout = message.expiry || 5
|
392
|
+
debug "RUN TASKS SUBMITTING #{message.name} job_id: #{job_id} #{message.payload.is_a?(Skynet::Task) ? 'task' + message.payload.task_id.to_s : ''}"
|
393
|
+
debug "RUN TASKS WORKER MESSAGE #{message.name} job_id: #{job_id}", message.to_a
|
394
|
+
mq.write_message(message,timeout * 5)
|
395
|
+
end
|
396
|
+
end
|
397
|
+
|
398
|
+
def gather_results(number_of_tasks, timeout=nil, description=nil)
|
399
|
+
debug "GATHER RESULTS job_id: #{job_id} - NOT AN ASYNC JOB"
|
400
|
+
results = {}
|
401
|
+
errors = {}
|
402
|
+
started_at = Time.now.to_i
|
403
|
+
|
173
404
|
begin
|
174
405
|
loop do
|
175
406
|
# debug "LOOKING FOR RESULT MESSAGE TEMPLATE"
|
176
|
-
result_message = mq.take_result(job_id,timeout * 2)
|
407
|
+
result_message = self.mq.take_result(job_id,timeout * 2)
|
408
|
+
ret_result = result_message.payload
|
177
409
|
|
178
|
-
ret_result = result_message.payload
|
179
410
|
if result_message.payload_type == :error
|
180
411
|
errors[result_message.task_id] = ret_result
|
181
412
|
error "ERROR RESULT TASK #{result_message.task_id} returned #{errors[result_message.task_id].inspect}"
|
182
413
|
else
|
183
|
-
|
184
|
-
debug "RESULT returned TASKID: #{result_message.task_id} #{
|
414
|
+
results[result_message.task_id] = ret_result
|
415
|
+
debug "RESULT returned TASKID: #{result_message.task_id} #{results[result_message.task_id].inspect}"
|
185
416
|
end
|
186
|
-
debug "RESULT collected: #{(
|
187
|
-
break if (
|
188
|
-
if (task_ids - (result.keys & errors.keys)).empty?
|
189
|
-
raise Skynet::Job::Error.new("WORKER ERROR #{description}, job_id: #{job_id} errors:#{errors.keys.size} out of #{task_ids.size} workers. #{errors.pretty_print_inspect}")
|
190
|
-
end
|
417
|
+
debug "RESULT collected: #{(results.keys + errors.keys).size}, remaining: #{(number_of_tasks - (results.keys + errors.keys).uniq.size)}"
|
418
|
+
break if (number_of_tasks - (results.keys + errors.keys).uniq.size) <= 0
|
191
419
|
end
|
192
|
-
rescue Skynet::RequestExpiredError => e
|
420
|
+
rescue Skynet::RequestExpiredError => e
|
421
|
+
local_mq_reset!
|
193
422
|
error "A WORKER EXPIRED or ERRORED, #{description}, job_id: #{job_id}"
|
194
423
|
if not errors.empty?
|
195
|
-
raise WorkerError.new("WORKER ERROR #{description}, job_id: #{job_id} errors:#{errors.keys.size} out of #{
|
424
|
+
raise WorkerError.new("WORKER ERROR #{description}, job_id: #{job_id} errors:#{errors.keys.size} out of #{number_of_tasks} workers. #{errors.pretty_print_inspect}")
|
196
425
|
else
|
197
426
|
raise Skynet::RequestExpiredError.new("WORKER ERROR, A WORKER EXPIRED! Did not get results or even errors back from all workers!")
|
198
427
|
end
|
199
428
|
end
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
429
|
+
local_mq_reset!
|
430
|
+
|
431
|
+
# ==========
|
432
|
+
# = FIXME Tricky one. Should we throw an exception if we didn't get all the results back, or should we keep going.
|
433
|
+
# = Maybe this is another needed option.
|
434
|
+
# ==========
|
435
|
+
# if not (errors.keys - results.keys).empty?
|
436
|
+
# raise WorkerError.new("WORKER ERROR #{description}, job_id: #{job_id} errors:#{errors.keys.size} out of #{number_of_tasks} workers. #{errors.pretty_print_inspect}")
|
437
|
+
# end
|
438
|
+
|
439
|
+
return nil if results.values.compact.empty?
|
440
|
+
return results.values
|
441
|
+
end
|
442
|
+
|
443
|
+
def local_mq_reset!
|
444
|
+
if use_local_queue?
|
445
|
+
local_mq.reset!
|
446
|
+
self.use_local_queue=false
|
215
447
|
end
|
216
|
-
@reduce_partitioner = klass if klass.constantize.respond_to?(:reduce_partitioner)
|
217
|
-
@map_partitioner = klass if klass.constantize.respond_to?(:map_partitioner)
|
218
448
|
end
|
449
|
+
|
219
450
|
|
220
|
-
def task_id
|
221
|
-
@task_id ||= get_unique_id(1)
|
222
|
-
end
|
223
|
-
|
224
|
-
# def run_master
|
225
|
-
# result = run_tasks(master_task,master_timeout,name)
|
226
|
-
# debug "MASTER RESULT #{self.name} job_id: #{self.job_id}", result
|
227
|
-
# result
|
228
|
-
# end
|
229
|
-
|
230
451
|
def master_task
|
231
452
|
@master_task ||= begin
|
232
453
|
raise Exception.new("No map provided") unless @map
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
:process
|
251
|
-
|
252
|
-
|
454
|
+
|
455
|
+
# Make sure to set single to false in our own Job object.
|
456
|
+
# We're just passing along whether they set us to single.
|
457
|
+
# If we were single, we'd never send off the master to be run externally.
|
458
|
+
@single = false
|
459
|
+
|
460
|
+
task = Skynet::Task.master_task(self)
|
461
|
+
end
|
462
|
+
end
|
463
|
+
|
464
|
+
def map_tasks
|
465
|
+
@map_tasks ||= begin
|
466
|
+
map_tasks = []
|
467
|
+
debug "RUN MAP 2.1 #{display_info} data size before partition: #{@map_data.size}" if @map_data.respond_to?(:size)
|
468
|
+
debug "RUN MAP 2.1 #{display_info} data before partition:", @map_data
|
469
|
+
|
470
|
+
task_options = {
|
471
|
+
:process => @map,
|
472
|
+
:name => map_name,
|
473
|
+
:map_or_reduce => :map,
|
474
|
+
:result_timeout => map_timeout,
|
475
|
+
:retry => map_retry || Skynet::CONFIG[:DEFAULT_MAP_RETRY]
|
476
|
+
}
|
477
|
+
|
478
|
+
if @map_data.is_a?(Array)
|
479
|
+
debug "RUN MAP 2.2 DATA IS Array #{display_info}"
|
480
|
+
num_mappers = @map_data.length < @mappers ? @map_data.length : @mappers
|
481
|
+
|
482
|
+
map_data = if @map_partitioner
|
483
|
+
@map_partitioner.call(@map_data,num_mappers)
|
484
|
+
else
|
485
|
+
Skynet::Partitioners::SimplePartitionData.reduce_partition(@map_data, num_mappers)
|
253
486
|
end
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
:
|
262
|
-
|
263
|
-
|
487
|
+
|
488
|
+
debug "RUN MAP 2.3 #{display_info} data size after partition: #{map_data.size}"
|
489
|
+
debug "RUN MAP 2.3 #{display_info} map data after partition:", map_data
|
490
|
+
elsif @map_data.is_a?(Enumerable)
|
491
|
+
debug "RUN MAP 2.2 DATA IS ENUMERABLE #{display_info} map_data_class: #{@map_data.class}"
|
492
|
+
map_data = @map_data
|
493
|
+
else
|
494
|
+
debug "RUN MAP 2.2 DATA IS NOT ARRAY OR ENUMERABLE #{display_info} map_data_class: #{@map_data.class}"
|
495
|
+
map_data = [ @map_data ]
|
496
|
+
end
|
497
|
+
Skynet::TaskIterator.new(task_options, map_data)
|
264
498
|
end
|
265
499
|
end
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
500
|
+
|
501
|
+
def reduce_tasks(partitioned_data)
|
502
|
+
@reduce_tasks ||= begin
|
503
|
+
task_options = {
|
504
|
+
:name => reduce_name,
|
505
|
+
:process => @reduce,
|
506
|
+
:map_or_reduce => :reduce,
|
507
|
+
:result_timeout => reduce_timeout,
|
508
|
+
:retry => reduce_retry || Skynet::CONFIG[:DEFAULT_REDUCE_RETRY]
|
509
|
+
}
|
510
|
+
Skynet::TaskIterator.new(task_options, partitioned_data)
|
511
|
+
end
|
270
512
|
end
|
271
513
|
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
# sometimes people want to run a master with just run. In this case we assume we have to set the data to the map_data
|
279
|
-
# XXX seems like a hack
|
514
|
+
def tasks_to_messages(tasks)
|
515
|
+
if tasks.is_a?(Skynet::TaskIterator)
|
516
|
+
tasks = tasks.to_a
|
517
|
+
elsif not tasks.is_a?(Array)
|
518
|
+
tasks = [tasks]
|
519
|
+
end
|
280
520
|
|
281
|
-
|
521
|
+
tasks.collect do |task|
|
522
|
+
Skynet::Message.new_task_message(task,self)
|
523
|
+
end
|
524
|
+
end
|
282
525
|
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
526
|
+
def master_retry
|
527
|
+
@master_retry || Skynet::CONFIG[:DEFAULT_MASTER_RETRY]
|
528
|
+
end
|
529
|
+
|
530
|
+
def map_retry
|
531
|
+
@map_retry || Skynet::CONFIG[:DEFAULT_MAP_RETRY]
|
532
|
+
end
|
533
|
+
|
534
|
+
def reduce_retry
|
535
|
+
@reduce_retry || Skynet::CONFIG[:DEFAULT_REDUCE_RETRY]
|
289
536
|
end
|
290
|
-
|
291
|
-
# Partition up starting data, create map tasks
|
292
|
-
def run_map
|
293
|
-
map_tasks = Array.new
|
294
|
-
debug "RUN MAP 2.1 #{display_info} data size before partition: #{@map_data.size}"
|
295
|
-
debug "RUN MAP 2.1 #{display_info} data before partition:", @map_data
|
296
|
-
if @map_data.class == Array
|
297
|
-
debug "RUN MAP 2.2 DATA IS Array #{display_info}"
|
298
|
-
num_mappers = @map_data.length < @map_tasks ? @map_data.length : @map_tasks
|
299
|
-
pre_map_data = Array.new
|
300
|
-
if @map_partitioner
|
301
|
-
pre_map_data = @map_partitioner.call(@map_data,num_mappers)
|
302
|
-
else
|
303
|
-
pre_map_data = Partitioner::simple_partition_data(@map_data, num_mappers)
|
304
|
-
end
|
305
|
-
debug "RUN MAP 2.3 #{display_info} data size after partition: #{pre_map_data.size}"
|
306
|
-
debug "RUN MAP 2.3 #{display_info} map data after partition:", pre_map_data
|
307
|
-
map_tasks = Array.new
|
308
|
-
|
309
|
-
(0..num_mappers - 1).each do |i|
|
310
|
-
map_tasks << Skynet::Task.new(
|
311
|
-
:task_id => get_unique_id(1),
|
312
|
-
:data => pre_map_data[i],
|
313
|
-
:process => @map,
|
314
|
-
:name => map_name,
|
315
|
-
:map_or_reduce => :map,
|
316
|
-
:result_timeout => result_timeout
|
317
|
-
)
|
318
|
-
end
|
319
|
-
|
320
|
-
# Run map tasks
|
321
|
-
#
|
322
|
-
elsif @map_data.is_a?(Enumerable)
|
323
|
-
debug "RUN MAP 2.2 DATA IS ENUMERABLE #{display_info} map_data_class: #{@map_data.class}"
|
324
|
-
each_method = @map_data.respond_to?(:next) ? :next : :each
|
325
|
-
@map_data.send(each_method) do |pre_map_data|
|
326
|
-
map_tasks << Skynet::Task.new(
|
327
|
-
:task_id => get_unique_id(1),
|
328
|
-
:data => pre_map_data,
|
329
|
-
:process => @map,
|
330
|
-
:name => map_name,
|
331
|
-
:map_or_reduce => :map,
|
332
|
-
:result_timeout => result_timeout
|
333
|
-
)
|
334
|
-
end
|
335
|
-
else
|
336
|
-
debug "RUN MAP 2.2 DATA IS NOT ARRAY OR ENUMERABLE #{display_info} map_data_class: #{@map_data.class}"
|
337
|
-
map_tasks = [
|
338
|
-
Skynet::Task.new(
|
339
|
-
:task_id => get_unique_id(1),
|
340
|
-
:data => @map_data,
|
341
|
-
:process => @map,
|
342
|
-
:name => map_name,
|
343
|
-
:map_or_reduce => :map,
|
344
|
-
:result_timeout => result_timeout
|
345
|
-
)
|
346
|
-
]
|
347
|
-
end
|
348
537
|
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
error "MAP FAILED #{display_info} #{e.class} #{e.message.inspect}"
|
353
|
-
return nil
|
354
|
-
end
|
538
|
+
def keep_map_tasks
|
539
|
+
@keep_map_tasks || Skynet::CONFIG[:DEFAULT_KEEP_MAP_TASKS]
|
540
|
+
end
|
355
541
|
|
356
|
-
|
542
|
+
def keep_reduce_tasks
|
543
|
+
@keep_reduce_tasks || Skynet::CONFIG[:DEFAULT_KEEP_REDUCE_TASKS]
|
544
|
+
end
|
545
|
+
|
546
|
+
def map_local?
|
547
|
+
return true if solo? or single?
|
548
|
+
return true if keep_map_tasks == true
|
549
|
+
# error "RUN MAP 2.4 BEFORE MAP #{display_info} KEEPMT:#{keep_map_tasks} DKMT:#{Skynet::CONFIG[:DEFAULT_KEEP_MAP_TASKS]} MDCLASS: #{map_tasks.data.class} #{(map_tasks.data.is_a?(Array) ? map_tasks.data.size : '')}"
|
550
|
+
return true if keep_map_tasks and map_tasks.data.is_a?(Array) and map_tasks.data.size <= keep_map_tasks
|
551
|
+
return false
|
552
|
+
end
|
357
553
|
|
358
|
-
|
554
|
+
def reduce_local?(reduce_tasks)
|
555
|
+
return true if solo? or single?
|
556
|
+
return true if keep_reduce_tasks == true
|
557
|
+
return true if keep_reduce_tasks and reduce_tasks.data.is_a?(Array) and reduce_tasks.data.size <= keep_reduce_tasks
|
558
|
+
return false
|
559
|
+
end
|
359
560
|
|
360
|
-
|
361
|
-
|
362
|
-
return post_map_data
|
561
|
+
def use_local_queue?
|
562
|
+
@use_local_queue
|
363
563
|
end
|
364
|
-
|
365
|
-
# Re-partition returning data for reduction, create reduce tasks
|
366
|
-
def run_reduce(post_map_data=nil)
|
367
|
-
return post_map_data unless post_map_data and @reduce
|
368
564
|
|
369
|
-
|
565
|
+
# async is true if the async flag is set and the job is not a 'single' job, or in solo mode.
|
566
|
+
# async only applies to whether we run the master locally and whether we poll for the result
|
567
|
+
def async?
|
568
|
+
@async and not (solo? or single? or local_master?)
|
569
|
+
end
|
570
|
+
|
571
|
+
def master?
|
572
|
+
async? or not local_master?
|
573
|
+
end
|
370
574
|
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
reduce_data = run_reduce_partitioner(post_map_data, num_reducers)
|
375
|
-
reduce_data.compact!
|
376
|
-
debug "RUN REDUCE 3.2 AFTER PARTITION #{display_info} num_reducers: #{reduce_data.length}"
|
377
|
-
debug "RUN REDUCE 3.2 AFTER PARTITION #{display_info} data:", reduce_data
|
378
|
-
reduce_tasks = Array.new
|
379
|
-
|
380
|
-
(0..reduce_data.length - 1).each do |i|
|
381
|
-
reduce_tasks << Skynet::Task.new(
|
382
|
-
:task_id => get_unique_id(1),
|
383
|
-
:data => reduce_data[i],
|
384
|
-
:name => reduce_name,
|
385
|
-
:process => @reduce,
|
386
|
-
:map_or_reduce => :reduce,
|
387
|
-
:result_timeout => result_timeout
|
388
|
-
)
|
389
|
-
end
|
390
|
-
reduce_tasks.compact! if reduce_tasks
|
391
|
-
|
392
|
-
debug "RUN REDUCE 3.3 CREATED REDUCE TASKS #{display_info}"#, reduce_tasks
|
393
|
-
|
394
|
-
# Reduce and return results
|
395
|
-
#
|
396
|
-
begin
|
397
|
-
results = run_tasks(reduce_tasks, reduce_timeout,reduce_name)
|
398
|
-
rescue WorkerError => e
|
399
|
-
error "REDUCE FAILED #{display_info} #{e.class} #{e.message.inspect}"
|
400
|
-
return nil
|
401
|
-
end
|
575
|
+
def local_master?
|
576
|
+
@local_master or solo?
|
577
|
+
end
|
402
578
|
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
579
|
+
def solo?
|
580
|
+
(@solo or CONFIG[:SOLO])
|
581
|
+
end
|
582
|
+
|
583
|
+
def single?
|
584
|
+
@single
|
585
|
+
end
|
586
|
+
|
587
|
+
def reset!
|
588
|
+
@map_tasks = nil
|
589
|
+
@reduce_tasks = nil
|
590
|
+
end
|
591
|
+
|
592
|
+
def to_h
|
593
|
+
if @map.kind_of?(Proc) or @reduce.kind_of?(Proc)
|
594
|
+
raise Skynet::Error.new("You have a Proc in your map or reduce. This can't be turned into a hash.")
|
408
595
|
end
|
409
|
-
|
410
|
-
|
411
|
-
|
596
|
+
hash = {}
|
597
|
+
FIELDS.each do |field|
|
598
|
+
hash[field] = self.send(field) if self.send(field)
|
599
|
+
end
|
600
|
+
hash
|
412
601
|
end
|
413
|
-
|
414
|
-
def
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
602
|
+
|
603
|
+
def task_id
|
604
|
+
@task_id ||= get_unique_id(1).to_i
|
605
|
+
end
|
606
|
+
|
607
|
+
def version
|
608
|
+
return 1 if solo?
|
609
|
+
@version ||= begin
|
610
|
+
@@worker_version ||= self.mq.get_worker_version || 1
|
611
|
+
@@worker_version
|
421
612
|
end
|
422
613
|
end
|
423
|
-
|
424
|
-
end ### END class Skynet::Job
|
425
|
-
|
426
|
-
class AsyncJob < Skynet::Job
|
427
|
-
|
428
|
-
## XXX Partitioning doesn't work yet!!!!!
|
429
614
|
|
430
|
-
def
|
431
|
-
|
432
|
-
super(opts)
|
615
|
+
def version=(v)
|
616
|
+
@version = v
|
433
617
|
end
|
618
|
+
|
619
|
+
def display_info
|
620
|
+
"#{name}, job_id: #{job_id}"
|
621
|
+
end
|
434
622
|
|
435
|
-
|
436
|
-
|
437
|
-
unless klass.class == String or klass.class == Class
|
438
|
-
raise BadMapOrReduceError.new("#{self.class}.map only accepts a class name")
|
439
|
-
end
|
440
|
-
klass = klass.to_s if klass.class == Symbol
|
441
|
-
@map = klass
|
623
|
+
def start_after=(time)
|
624
|
+
@start_after = (time.is_a?(Time) ? time.to_i : time)
|
442
625
|
end
|
626
|
+
|
627
|
+
def map_data=(map_data)
|
628
|
+
reset!
|
629
|
+
@map_data = map_data
|
630
|
+
end
|
631
|
+
|
632
|
+
def map=(map)
|
633
|
+
reset!
|
634
|
+
return unless map
|
635
|
+
if map.class == String or map.class == Class
|
636
|
+
@map = map.to_s
|
637
|
+
elsif map.is_a?(Proc)
|
638
|
+
@map = map
|
639
|
+
else
|
640
|
+
raise BadMapOrReduceError.new("#{self.class}.map accepts a class name or a proc. Got #{map}")
|
641
|
+
end
|
642
|
+
end
|
643
|
+
|
644
|
+
def reduce=(reduce)
|
645
|
+
reset!
|
646
|
+
return unless reduce
|
647
|
+
if reduce.class == String or reduce.class == Class
|
648
|
+
@reduce = reduce.to_s
|
649
|
+
elsif reduce.is_a?(Proc)
|
650
|
+
@reduce = reduce
|
651
|
+
else
|
652
|
+
raise BadMapOrReduceError.new("#{self.class}.reduce accepts a class name or a proc. Got #{reduce}")
|
653
|
+
end
|
654
|
+
end
|
443
655
|
|
444
|
-
def
|
656
|
+
def map_reduce_class=(klass)
|
657
|
+
reset!
|
445
658
|
unless klass.class == String or klass.class == Class
|
446
|
-
raise BadMapOrReduceError.new("#{self.class}.
|
659
|
+
raise BadMapOrReduceError.new("#{self.class}.map_reduce only accepts a class name: #{klass} #{klass.class}")
|
447
660
|
end
|
448
|
-
klass = klass.to_s
|
449
|
-
@
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
else
|
456
|
-
results = run_tasks(master_task,master_timeout,name)
|
457
|
-
self.job_id
|
661
|
+
klass = klass.to_s
|
662
|
+
@map = klass
|
663
|
+
self.name ||= "#{klass} MASTER"
|
664
|
+
self.map_name ||= "#{klass} MAP"
|
665
|
+
if klass.constantize.respond_to?(:reduce)
|
666
|
+
@reduce ||= klass
|
667
|
+
self.reduce_name ||= "#{klass} REDUCE"
|
458
668
|
end
|
669
|
+
@reduce_partitioner ||= klass if klass.constantize.respond_to?(:reduce_partition)
|
670
|
+
@map_partitioner ||= klass if klass.constantize.respond_to?(:map_partitioner)
|
459
671
|
end
|
460
672
|
|
461
|
-
def
|
462
|
-
|
463
|
-
|
464
|
-
set_version
|
465
|
-
job = Skynet::Job.new(
|
466
|
-
:map_timeout => map_timeout,
|
467
|
-
:reduce_timeout => reduce_timeout,
|
468
|
-
:job_id => task_id,
|
469
|
-
:map_data => @map_data,
|
470
|
-
:map_name => map_name || name,
|
471
|
-
:reduce_name => reduce_name || name,
|
472
|
-
:map => @map,
|
473
|
-
:map_partitioner => @map_partitioner,
|
474
|
-
:reduce => @reduce,
|
475
|
-
:reduce_partitioner => @reduce_partitioner,
|
476
|
-
:map_tasks => @map_tasks,
|
477
|
-
:reduce_tasks => @reduce_tasks,
|
478
|
-
:name => @name,
|
479
|
-
:version => version,
|
480
|
-
:result_timeout => result_timeout,
|
481
|
-
:master_result_timeout => master_result_timeout,
|
482
|
-
:solo => solo,
|
483
|
-
:single => single
|
484
|
-
)
|
485
|
-
@single = false
|
486
|
-
|
487
|
-
task = Skynet::Task.new(
|
488
|
-
:task_id => task_id,
|
489
|
-
:data => nil,
|
490
|
-
:process => job.to_h,
|
491
|
-
:map_or_reduce => :master,
|
492
|
-
:name => self.name,
|
493
|
-
:result_timeout => master_result_timeout
|
494
|
-
)
|
495
|
-
end
|
673
|
+
def run_master
|
674
|
+
error "run_master has been deprecated, please use run"
|
675
|
+
run(:local_master => false)
|
496
676
|
end
|
497
677
|
|
498
|
-
def
|
499
|
-
if
|
500
|
-
|
678
|
+
def mq
|
679
|
+
if use_local_queue?
|
680
|
+
local_mq
|
501
681
|
else
|
502
|
-
|
682
|
+
@mq ||= Skynet::MessageQueue.new
|
503
683
|
end
|
504
684
|
end
|
505
|
-
|
506
|
-
|
685
|
+
|
686
|
+
def local_mq
|
687
|
+
@local_mq ||= LocalMessageQueue.new
|
688
|
+
end
|
507
689
|
|
508
|
-
|
509
|
-
|
510
|
-
|
690
|
+
def self.mq
|
691
|
+
Skynet::MessageQueue.new
|
692
|
+
end
|
693
|
+
|
694
|
+
end ### END class Skynet::Job
|
695
|
+
end
|
696
|
+
|
697
|
+
class Skynet::AsyncJob < Skynet::Job
|
698
|
+
# Skynet::AsyncJob is for Skynet jobs you want to run asyncronously.
|
699
|
+
# Normally when you run a Skynet::Job it blocks until the job is complete.
|
700
|
+
# Running an Async job merely returns a job_id which can be used later to retrieve the results.
|
701
|
+
# See Skynet::Job for full documentation
|
511
702
|
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
703
|
+
def initialize(options = {})
|
704
|
+
options[:async] = true
|
705
|
+
options[:local_master] = false
|
706
|
+
super(options)
|
707
|
+
end
|
708
|
+
|
709
|
+
def map=(klass)
|
710
|
+
unless klass.class == String or klass.class == Class
|
711
|
+
raise BadMapOrReduceError.new("#{self.class}.map only accepts a class name")
|
516
712
|
end
|
713
|
+
@map = klass.to_s
|
714
|
+
end
|
517
715
|
|
518
|
-
|
519
|
-
|
520
|
-
|
716
|
+
def reduce=(klass)
|
717
|
+
unless klass.class == String or klass.class == Class
|
718
|
+
raise BadMapOrReduceError.new("#{self.class}.reduce only accepts a class name")
|
521
719
|
end
|
720
|
+
@reduce = klass.to_s
|
721
|
+
end
|
722
|
+
end # class Skynet::AsyncJob
|
522
723
|
|
523
|
-
|
524
|
-
|
724
|
+
class Skynet::Job::LocalMessageQueue
|
725
|
+
include SkynetDebugger
|
726
|
+
|
727
|
+
attr_reader :messages, :results
|
525
728
|
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
partitioned_data << [datum]
|
535
|
-
end
|
536
|
-
elsif (data.length >= partitions * 2)
|
537
|
-
# Use quicker but less "fair" method
|
538
|
-
size = data.length / partitions
|
729
|
+
def initialize
|
730
|
+
@messages = []
|
731
|
+
@results = []
|
732
|
+
end
|
733
|
+
|
734
|
+
def get_worker_version
|
735
|
+
1
|
736
|
+
end
|
539
737
|
|
540
|
-
|
541
|
-
|
542
|
-
|
738
|
+
def take_result(job_id,timeout=nil)
|
739
|
+
raise Skynet::RequestExpiredError.new if @messages.empty?
|
740
|
+
run_message(@messages.shift)
|
741
|
+
end
|
543
742
|
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
else
|
548
|
-
# Slower method, but partitions evenly
|
549
|
-
partitions = (data.size < partitions ? data.size : partitions)
|
550
|
-
(0..partitions - 1).each { |i| partitioned_data[i] = Array.new }
|
551
|
-
|
552
|
-
data.each_with_index do |datum, i|
|
553
|
-
partitioned_data[i % partitions] << datum
|
554
|
-
end
|
555
|
-
end
|
556
|
-
|
557
|
-
partitioned_data
|
558
|
-
end
|
743
|
+
def write_message(message,timeout=nil)
|
744
|
+
@messages << message
|
745
|
+
end
|
559
746
|
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
lambda do |post_map_data, new_partitions|
|
747
|
+
def empty?
|
748
|
+
@messages.empty?
|
749
|
+
end
|
564
750
|
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
575
|
-
|
751
|
+
def in_use?
|
752
|
+
(not empty?)
|
753
|
+
end
|
754
|
+
|
755
|
+
def reset!
|
756
|
+
@messages = []
|
757
|
+
@results = []
|
758
|
+
end
|
759
|
+
|
760
|
+
def run_message(message)
|
761
|
+
result = nil
|
762
|
+
(message.retry + 1).times do
|
763
|
+
task = message.payload
|
764
|
+
debug "RUN TASKS LOCALLY SUBMITTING #{message.name} task #{task.task_id}", task
|
765
|
+
begin
|
766
|
+
result = task.run
|
767
|
+
break
|
768
|
+
rescue Skynet::Task::TimeoutError => e
|
769
|
+
result = e
|
770
|
+
error "Skynet::Job::LocalMessageQueue Task timed out while executing #{e.inspect} #{e.backtrace.join("\n")}"
|
771
|
+
next
|
772
|
+
rescue Exception => e
|
773
|
+
error "Skynet::Job::LocalMessageQueue :#{__LINE__} #{e.inspect} #{e.backtrace.join("\n")}"
|
774
|
+
result = e
|
775
|
+
next
|
576
776
|
end
|
577
777
|
end
|
778
|
+
message.result_message(result)
|
779
|
+
end
|
780
|
+
end # class LocalMessageQueue
|
781
|
+
|
782
|
+
class Skynet::TaskIterator
|
783
|
+
include SkynetDebugger
|
784
|
+
include Skynet::GuidGenerator
|
578
785
|
|
786
|
+
class Error < StandardError
|
787
|
+
end
|
579
788
|
|
580
|
-
|
581
|
-
# and ensures that all arrays sharing that key go into the same partition.
|
582
|
-
#
|
583
|
-
def self.array_data_split_by_first_entry
|
584
|
-
lambda do |partitioned_data, new_partitions|
|
585
|
-
partitions = Array.new
|
586
|
-
(0..new_partitions - 1).each { |i| partitions[i] = Array.new }
|
587
|
-
|
588
|
-
partitioned_data.each do |partition|
|
589
|
-
partition.each do |array|
|
590
|
-
next unless array.class == Array and array.size == 2
|
591
|
-
if array[0].kind_of?(Fixnum)
|
592
|
-
key = array[0]
|
593
|
-
else
|
594
|
-
key = 0
|
595
|
-
array[0].each_byte { |c| key += c }
|
596
|
-
end
|
597
|
-
partitions[key % new_partitions] << array
|
598
|
-
end
|
599
|
-
end
|
789
|
+
include Enumerable
|
600
790
|
|
601
|
-
|
602
|
-
|
791
|
+
attr_accessor :task_options, :data
|
792
|
+
|
793
|
+
def initialize(task_options, data)
|
794
|
+
@task_options = task_options
|
795
|
+
@data = data
|
796
|
+
end
|
797
|
+
|
798
|
+
def first
|
799
|
+
if data.respond_to?(:first)
|
800
|
+
@first ||= Skynet::Task.new(task_options.merge(:data => data.first, :task_id => get_unique_id(1).to_i))
|
801
|
+
else
|
802
|
+
raise Error.new("#{data.class} does not implement 'first'")
|
603
803
|
end
|
804
|
+
end
|
805
|
+
|
806
|
+
def size
|
807
|
+
if data.respond_to?(:size)
|
808
|
+
data.size
|
809
|
+
else
|
810
|
+
raise Error.new("#{data.class} does not implement 'size'")
|
811
|
+
end
|
812
|
+
end
|
604
813
|
|
814
|
+
def [](index)
|
815
|
+
if data.respond_to?(:[])
|
816
|
+
Skynet::Task.new(task_options.merge(:data => data[index], :task_id => get_unique_id(1).to_i))
|
817
|
+
else
|
818
|
+
raise Error.new("#{data.class} does not implement '[]'")
|
819
|
+
end
|
820
|
+
end
|
821
|
+
|
822
|
+
def each_method
|
823
|
+
each_method = data.respond_to?(:next) ? :next : :each
|
824
|
+
end
|
825
|
+
|
826
|
+
def to_a
|
827
|
+
self.collect { |task| task }
|
605
828
|
end
|
606
829
|
|
607
|
-
|
830
|
+
def each
|
831
|
+
iteration = 0
|
832
|
+
data.send(each_method) do |task_data|
|
833
|
+
task = nil
|
834
|
+
if @first and iteration == 0
|
835
|
+
task = @first
|
836
|
+
else
|
837
|
+
task = Skynet::Task.new(task_options.merge(:data => task_data, :task_id => (get_unique_id(1).to_i)))
|
838
|
+
@first = task if iteration == 0
|
839
|
+
end
|
840
|
+
iteration += 1
|
841
|
+
yield task
|
842
|
+
end
|
843
|
+
end
|
844
|
+
end # class TaskIterator
|
845
|
+
|
846
|
+
|
847
|
+
# require 'ruby2ruby' # XXX this will break unless people have the fix to Ruby2Ruby
|
848
|
+
##### ruby2ruby fix from ruby2ruby.rb ############
|
849
|
+
### XXX This is bad. Some people rely on an exception being thrown if a method is missing! BULLSHIT!
|
850
|
+
# class NilClass # Objective-C trick
|
851
|
+
# def method_missing(msg, *args, &block)
|
852
|
+
# nil
|
853
|
+
# end
|
854
|
+
# end
|
855
|
+
##############################
|