skynet 0.9.1 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. data/History.txt +99 -0
  2. data/Manifest.txt +10 -9
  3. data/README.txt +74 -7
  4. data/app_generators/skynet_install/skynet_install_generator.rb +26 -22
  5. data/app_generators/skynet_install/templates/migration.rb +11 -5
  6. data/app_generators/skynet_install/templates/skynet +25 -12
  7. data/app_generators/skynet_install/templates/skynet_schema.sql +56 -0
  8. data/bin/skynet +26 -2
  9. data/bin/skynet_install +24 -0
  10. data/bin/skynet_tuplespace_server +13 -0
  11. data/config/hoe.rb +1 -0
  12. data/lib/skynet.rb +3 -0
  13. data/lib/skynet/mapreduce_helper.rb +74 -0
  14. data/lib/skynet/message_queue_adapters/mysql.rb +225 -172
  15. data/lib/skynet/message_queue_adapters/tuple_space.rb +31 -16
  16. data/lib/skynet/skynet_active_record_extensions.rb +78 -46
  17. data/lib/skynet/skynet_config.rb +162 -23
  18. data/lib/skynet/skynet_console.rb +23 -10
  19. data/lib/skynet/skynet_console_helper.rb +61 -58
  20. data/lib/skynet/skynet_job.rb +741 -493
  21. data/lib/skynet/skynet_launcher.rb +5 -1
  22. data/lib/skynet/skynet_manager.rb +106 -49
  23. data/lib/skynet/skynet_message.rb +169 -174
  24. data/lib/skynet/skynet_message_queue.rb +29 -16
  25. data/lib/skynet/skynet_partitioners.rb +92 -0
  26. data/lib/skynet/skynet_ruby_extensions.rb +3 -4
  27. data/lib/skynet/skynet_task.rb +61 -19
  28. data/lib/skynet/skynet_tuplespace_server.rb +0 -2
  29. data/lib/skynet/skynet_worker.rb +73 -51
  30. data/lib/skynet/version.rb +1 -1
  31. data/test/test_active_record_extensions.rb +138 -0
  32. data/test/test_helper.rb +6 -0
  33. data/test/{mysql_message_queue_adaptor_test.rb → test_mysql_message_queue_adapter.rb} +94 -30
  34. data/test/test_skynet.rb +11 -11
  35. data/test/test_skynet_install_generator.rb +0 -4
  36. data/test/test_skynet_job.rb +717 -0
  37. data/test/test_skynet_manager.rb +142 -0
  38. data/test/test_skynet_message.rb +229 -0
  39. data/test/test_skynet_task.rb +24 -0
  40. data/test/{tuplespace_message_queue_test.rb → test_tuplespace_message_queue.rb} +25 -30
  41. data/website/index.html +56 -16
  42. data/website/index.txt +55 -25
  43. data/website/template.rhtml +1 -1
  44. metadata +29 -13
  45. data/app_generators/skynet_install/templates/skynet_console +0 -16
  46. data/bin/skynet_console +0 -9
  47. data/sometest.rb +0 -23
  48. data/test/all_models_test.rb +0 -139
  49. data/test/skynet_manager_test.rb +0 -107
  50. data/test/skynet_message_test.rb +0 -42
  51. data/tmtags +0 -1242
@@ -4,6 +4,9 @@ class Skynet
4
4
  require 'rubygems'
5
5
  require 'optparse'
6
6
  require 'skynet'
7
+ require "skynet_console_helper"
8
+ require "irb/completion"
9
+ require "irb"
7
10
 
8
11
  irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
9
12
 
@@ -13,22 +16,32 @@ class Skynet
13
16
  }
14
17
 
15
18
  OptionParser.new do |opt|
16
- opt.banner = "Usage: skynet_console [options]"
19
+ opt.banner = "Usage: skynet console [options]"
17
20
  opt.on("--irb=[#{irb}]", 'Invoke a different irb.') { |v| options[:irb] = v }
18
21
  opt.on('-r', '--required LIBRARY', 'Require the specified libraries. To include multiple libraries, include multiple -r options. ie. -r skynet -r fileutils') do |v|
19
22
  options[:required_libs] << File.expand_path(v)
20
23
  end
21
24
  opt.parse!(ARGV)
22
25
  end
23
-
24
- libs << "irb/completion"
25
- libs << "rubygems"
26
- libs << "skynet"
27
- libs << "skynet_console_helper"
28
- libs += options[:required_libs]
29
- cmd = "#{options[:irb]} -r #{libs.join(" -r ")} --simple-prompt"
30
-
31
- exec cmd
26
+
27
+ options[:required_libs] + libs
28
+ options[:required_libs].uniq.each do |lib|
29
+ require lib
30
+ end
31
+
32
+ IRB.setup(Skynet::CONFIG[:LAUNCHER_PATH])
33
+ IRB.conf[:PROMPT_MODE] = :SIMPLE
34
+ irb = IRB::Irb.new()
35
+ IRB.conf[:MAIN_CONTEXT] = irb.context
36
+ irb.context.workspace.main.extend Skynet::ConsoleHelper
37
+
38
+ trap("SIGINT") do
39
+ irb.signal_handle
40
+ end
41
+
42
+ catch(:IRB_EXIT) do
43
+ irb.eval_input
44
+ end
32
45
  end
33
46
  end
34
47
  end
@@ -1,59 +1,62 @@
1
-
2
- def mq
3
- @mq ||= Skynet::MessageQueue.new
4
- end
5
-
6
- def stats
7
- mq.stats
8
- end
9
-
10
- def increment_worker_version
11
- mq.increment_worker_version
12
- end
13
-
14
- def get_worker_version
15
- mq.get_worker_version
16
- end
17
-
18
- def set_worker_version(*args)
19
- mq.set_worker_version(*args)
20
- end
21
-
22
- def manager
23
- @manager ||= DRbObject.new(nil,Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL])
1
+ module Skynet::ConsoleHelper
2
+ # All of these commands can be run at the 'skynet console'.
3
+
4
+ def mq
5
+ @mq ||= Skynet::MessageQueue.new
6
+ end
7
+
8
+ def stats
9
+ mq.stats
10
+ end
11
+
12
+ def increment_worker_version
13
+ mq.increment_worker_version
14
+ end
15
+
16
+ def get_worker_version
17
+ mq.get_worker_version
18
+ end
19
+
20
+ def set_worker_version(*args)
21
+ mq.set_worker_version(*args)
22
+ end
23
+
24
+ def manager
25
+ @manager ||= DRbObject.new(nil,Skynet::CONFIG[:SKYNET_LOCAL_MANAGER_URL])
26
+ end
27
+
28
+ def add_lib(lib)
29
+ manager.required_libs << File.expand_path(lib)
30
+ manager.restart_workers
31
+ end
32
+
33
+ def restart_workers
34
+ manager.restart_workers
35
+ end
36
+
37
+ def add_workers(num)
38
+ manager.add_workers(num)
39
+ end
40
+
41
+ def remove_workers(num)
42
+ manager.remove_workers(num)
43
+ end
44
+
45
+ # ===============
46
+ # = Doesnt work =
47
+ # ===============
48
+ # def help
49
+ # puts <<-HELP
50
+ # mq
51
+ # stats
52
+ # increment_worker_version
53
+ # get_worker_version
54
+ # set_worker_version(version)
55
+ # manager
56
+ # add_lib(library_to_include) -- forces a restart
57
+ # restart_workers
58
+ # add_workers(number_of_workers)
59
+ # remove_workers(number_of_workers)
60
+ # HELP
61
+ # end
24
62
  end
25
-
26
- def add_lib(lib)
27
- manager.required_libs << File.expand_path(lib)
28
- manager.restart_workers
29
- end
30
-
31
- def restart_workers
32
- manager.restart_workers
33
- end
34
-
35
- def add_workers(num)
36
- manager.add_workers(num)
37
- end
38
-
39
- def remove_workers(num)
40
- manager.remove_workers(num)
41
- end
42
-
43
- # ===============
44
- # = Doesnt work =
45
- # ===============
46
- # def help
47
- # puts <<-HELP
48
- # mq
49
- # stats
50
- # increment_worker_version
51
- # get_worker_version
52
- # set_worker_version(version)
53
- # manager
54
- # add_lib(library_to_include) -- forces a restart
55
- # restart_workers
56
- # add_workers(number_of_workers)
57
- # remove_workers(number_of_workers)
58
- # HELP
59
- # end
@@ -1,607 +1,855 @@
1
- # require 'ruby2ruby' # XXX this will break unless people have the fix to Ruby2Ruby
2
- ##### ruby2ruby fix from ruby2ruby.rb ############
3
- ### XXX This is bad. Some people rely on an exception being thrown if a method is missing! BULLSHIT!
4
- # class NilClass # Objective-C trick
5
- # def method_missing(msg, *args, &block)
6
- # nil
7
- # end
8
- # end
9
- ##############################
10
-
11
- # Users should create instances of this class. Rather than subclassing,
12
- # jobs are specialized by assigning lambdas to map, reduce, and partition.
13
- # This allows the instance to easily create sub-tasks and marshal the map
14
- # and reduce code for sending to workers.
15
- #
16
-
17
1
  class Skynet
2
+ # Skynet::Job is the main interface to Skynet. You create a job object giving
3
+ # it the starting data (map_data), along with what class has the map/reduce
4
+ # functions in it. Even though Skynet is distributed, when you call #run on
5
+ # a plain Skynet::Job, it will still block in your current process until it has completed
6
+ # your task. If you want to go on to do other things you'll want to pass :async => true
7
+ # when creating a new job. Then later call job.results to retrieve your results.
8
+ #
9
+ # There are also many global configuration options which can be controlled through Skynet::CONFIG
10
+ #
11
+ # Example Usage:
12
+ #
13
+ # class MapReduceTest
14
+ #
15
+ # def self.run
16
+ # job = Skynet::Job.new(
17
+ # :mappers => 2,
18
+ # :reducers => 1,
19
+ # :map_reduce_class => self,
20
+ # :map_data => [OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 2}),OpenStruct.new({:created_by => 3})]
21
+ # )
22
+ # results = job.run
23
+ # end
24
+ #
25
+ # def self.map(profiles)
26
+ # result = Array.new
27
+ # profiles.each do |profile|
28
+ # result << [profile.created_by, 1] if profile.created_by
29
+ # end
30
+ # result
31
+ # end
32
+ #
33
+ # def self.reduce(pairs)
34
+ # totals = Hash.new
35
+ # pairs.each do |pair|
36
+ # created_by, count = pair[0], pair[1]
37
+ # totals[created_by] ||= 0
38
+ # totals[created_by] += count
39
+ # end
40
+ # totals
41
+ # end
42
+ # end
43
+ #
44
+ # MapReduceTest.run
45
+ #
46
+ # You might notice that self.map and self.reduce both accept Arrays. If you do not want to deal with
47
+ # getting arrays of map_data or reduce_data, you can include MapreduceHelper into your class and then
48
+ # implement self.map_each and self.reduce_each methods. The included self.map and self.reduce methods
49
+ # will handle iterating over the map_data and reduce_data, passing each element to your map_each and
50
+ # reduce_each methods respectively. They will also handle error handling within that loop to make sure
51
+ # even if a single map or reduce fails, processing will continue. If you do not want processing to
52
+ # continue if a map fails, do not use the MapreduceHelper mixin.
53
+ #
54
+ # There are many other options to control various defaults and timeouts.
18
55
  class Job
19
56
  include SkynetDebugger
20
57
  include Skynet::GuidGenerator
21
58
 
22
- class WorkerError < Skynet::Error
23
- end
59
+ class WorkerError < Skynet::Error; end
24
60
 
25
- class BadMapOrReduceError < Skynet::Error
26
- end
61
+ class BadMapOrReduceError < Skynet::Error; end
27
62
 
28
- class Error < Skynet::Error
29
- end
63
+ class Error < Skynet::Error; end
30
64
 
31
- @@svn_rev = nil
32
65
  @@worker_ver = nil
33
- @@log = nil
34
66
 
35
- FIELDS = [:map_tasks, :reduce_tasks, :silent, :name, :map_timeout, :map_data, :job_id,
36
- :reduce_timeout, :master_timeout, :master, :map_name, :reduce_name, :async,
37
- :master_result_timeout, :result_timeout, :start_after, :solo, :single,
38
- :map, :map_partitioner, :reduce, :reduce_partitioner, :single
67
+ FIELDS = [:queue_id, :mappers, :reducers, :silent, :name, :map_timeout, :map_data, :job_id,
68
+ :reduce_timeout, :master_timeout, :map_name, :reduce_name,
69
+ :master_result_timeout, :result_timeout, :start_after, :solo, :single, :version,
70
+ :map, :map_partitioner, :reduce, :reduce_partition, :map_reduce_class,
71
+ :master_retry, :map_retry, :reduce_retry,
72
+ :keep_map_tasks, :keep_reduce_tasks,
73
+ :local_master, :async
39
74
  ]
40
75
 
41
76
  FIELDS.each do |method|
42
- attr_accessor method
43
- end
77
+ if [:map_reduce_class, :version, :map, :reduce, :map_data, :start_after].include?(method)
78
+ attr_reader method
79
+ elsif [:master_retry, :map_retry, :reduce_retry,:keep_map_tasks, :keep_reduce_tasks].include?(method)
80
+ attr_writer method
81
+ else
82
+ attr_accessor method
83
+ end
84
+ end
85
+
86
+ attr_accessor :use_local_queue
87
+
88
+ Skynet::CONFIG[:JOB_DEFAULTS] = {
89
+ :queue_id => 0,
90
+ :mappers => 2,
91
+ :reducers => 1,
92
+ :map_timeout => 60,
93
+ :reduce_timeout => 60,
94
+ :master_timeout => 60,
95
+ :result_timeout => 1200,
96
+ :start_after => 0,
97
+ :master_result_timeout => 1200,
98
+ :local_master => true
99
+ }
44
100
 
45
-
46
101
  def self.debug_class_desc
47
102
  "JOB"
48
103
  end
49
104
 
50
- def initialize(opts = {})
51
- @name = opts[:name]
52
- @map_name = opts[:map_name]
53
- @reduce_name = opts[:reduce_name]
54
- @silent = opts[:silent]
55
- @master = opts[:master]
56
- @async = opts[:async]
57
- @solo = opts[:solo]
58
- @single = opts[:single]
59
- @version = opts[:version] if opts[:version]
60
- @map_tasks = opts[:map_tasks] || 2
61
- @reduce_tasks = opts[:reduce_tasks] || 1
62
- @map_timeout = opts[:map_timeout] || 60
63
- @reduce_timeout = opts[:reduce_timeout] || 60
64
- @master_timeout = opts[:master_timeout] || 60
65
- @result_timeout = opts[:result_timeout] || 1200
66
- @start_after = opts[:start_after] || 0
67
- @master_result_timeout = opts[:master_result_timeout] || 1200
68
-
69
- @map_data = opts[:map_data]
70
- if opts[:map_reduce_class]
71
- self.map_reduce_class = opts[:map_reduce_class]
72
- else
73
- self.map = opts[:map] if opts[:map]
74
- self.reduce = opts[:reduce] if opts[:reduce]
75
- end
105
+ # Most of the time you will merely call #new(options) and then #run on the returned object.
106
+ #
107
+ # Options are:
108
+ # <tt>:local_master</tt> BOOL (DEFAULT true)
109
+ # By default, your Skynet::Job will act as the master for your map/reduce job, doling out
110
+ # tasks, waiting for other workers to complete and return their results and dealing with
111
+ # merging and partitioning the data. If you call #run in async mode, another worker will handle
112
+ # being the master for your job without blocking. If you run :async => false, :local_master => false
113
+ # Skynet will let another worker be the master for your job, but will block waiting for the
114
+ # final results. The benefit of this is that if your process dies, the Job will continue to
115
+ # run remotely.
116
+ #
117
+ # <tt>:async</tt> BOOL (DEFAULT false)
118
+ # If you run in async mode, another worker will handle being the master for your job without blocking.
119
+ # You can not pass :local_master => true, :async => true since the only way to allow your
120
+ # job to run asyncronously is to have a remote_master.
121
+ #
122
+ # <tt>:map_data</tt>(Array or Enumerable)
123
+ # map_data should be an Array or Enumerable that data Skynet::Job will split up
124
+ # and distribute among your workers. You can stream data to Skynet::Job by passing
125
+ # an Enumerable that implements next or each.
126
+ #
127
+ # <tt>:map_reduce_class</tt>(Class or Class Name)
128
+ # Skynet::Job will look for class methods named self.map, self.reduce, self.map_partitioner,
129
+ # self.reduce_partition in your map_reduce_class. The only method requires is self.map.
130
+ # Each of these methods must accept an array. Examples above.
131
+ #
132
+ # <tt>:map</tt>(Class Name)
133
+ # You can pass a classname, or a proc. If you pass a classname, Job will look for a method
134
+ # called self.map in that class.
135
+ # WARNING: Passing a proc does not work right now.
136
+ #
137
+ # <tt>:reduce</tt>(Class Name)
138
+ # You can pass a classname, or a proc. If you pass a classname, Job will look for a method
139
+ # called self.reduce in that class.
140
+ # WARNING: Passing a proc does not work right now.
141
+ #
142
+ # <tt>:reduce_partition</tt>(Class Name)
143
+ # You can pass a classname, or a proc. If you pass a classname, Job will look for a method
144
+ # called self.reduce_partition in that class.
145
+ # WARNING: Passing a proc does not work right now.
146
+ #
147
+ # <tt>:mappers</tt> Fixnum
148
+ # The number of mappers to partition map data for.
149
+ #
150
+ # <tt>:reducers</tt> Fixnum
151
+ # The number of reducers to partition the returned map_data for.
152
+ #
153
+ # <tt>:master_retry</tt> Fixnum
154
+ # If the master fails for any reason, how many times should it be retried? You can also set
155
+ # Skynet::CONFIG[:DEFAULT_MASTER_RETRY] (DEFAULT 0)
156
+ #
157
+ # <tt>:map_retry</tt> Fixnum
158
+ # If a map task fails for any reason, how many times should it be retried? You can also set
159
+ # Skynet::CONFIG[:DEFAULT_MAP_RETRY] (DEFAULT 3)
160
+ #
161
+ # <tt>:reduce_retry</tt> Fixnum
162
+ # If a reduce task fails for any reason, how many times should it be retried? You can also set
163
+ # Skynet::CONFIG[:DEFAULT_REDUCE_RETRY] (DEFAULT 3)
164
+ #
165
+ # <tt>:master_timeout</tt>, <tt>:map_timeout</tt>, <tt>:reduce_timeout</tt>, <tt>master_result_timeout</tt>, <tt>result_timeout</tt>
166
+ # These control how long skynet should wait for particular actions to be finished.
167
+ # The master_timeout controls how long the master should wait for ALL map/reduce tasks ie. the entire job to finish.
168
+ # The master_result_timeout controls how long the final result should wait in the queue before being expired.
169
+ # The map and reduce timeouts control how long individual map and reduce tasks shoudl take.
170
+ #
171
+ # <tt>:single</tt> BOOL
172
+ # By default the master task distributes the map and reduce tasks to other workers.
173
+ # In single mode the master will take care of the map and reduce tasks by itself.
174
+ # This is handy when you really want to just perform some single action asyncronously.
175
+ # In this case you're merely using Skynet to postpone some action. In single mode, the
176
+ # first worker that picks up your task will just complete it as opposed to trying to distribute
177
+ # it to another worker.
178
+ #
179
+ # <tt>:start_after</tt> Time or Time.to_i
180
+ # Do not start job until :start_after has passed
181
+ #
182
+ # <tt>:queue</tt> String
183
+ # Which queue should this Job go in to? The queue provided is merely used to
184
+ # determine the queue_id.
185
+ # Queues are defined in Skynet::CONFIG[:MESSAGE_QUEUES]
186
+ #
187
+ # <tt>:queue_id</tt> Fixnum (DEFAULT 0)
188
+ # Which queue should this Job go in to?
189
+ # Queues are defined in Skynet::CONFIG[:MESSAGE_QUEUES]
190
+ #
191
+ # <tt>:solo</tt> BOOL
192
+ # One normally turns solo mode in in Skynet::Config using Skynet::CONFIG[:SOLO] = true
193
+ # In solo mode, Skynet jobs do not add items to a Skynet queue. Instead they do all
194
+ # work in place. It's like a Skynet simulation mode. It will complete all tasks
195
+ # without Skynet running. Great for testing. You can also wrap code blocks in
196
+ # Skynet.solo {} to run that code in solo mode.
197
+ #
198
+ # <tt>:version</tt> Fixnum
199
+ # If you do not provide a version the current worker version will be used.
200
+ # Skynet workers start at a specific version and only look for jobs that match that version.
201
+ # A worker will continue looking for jobs at that version until there are no more jobs left on
202
+ # the queue for that version. At that time, the worker will check to see if there is a new version.
203
+ # If there is, it will restart itself at the new version (assuming you had already pushed code to
204
+ # said workers.)
205
+ # To retrieve the current version, set the current version or increment the current version, see
206
+ # Skynet::Job.set_worker_version, Skynet::Job.get_worker_version, Skynet::Job.increment_worker_version
207
+ #
208
+ # <tt>:name</tt>, <tt>:map_name</tt>, <tt>:reduce_name</tt>
209
+ # These name methods are merely for debugging while watching the Skynet logs or the Skynet queue.
210
+ # If you do not supply names, it will try and provide sensible ones based on your class names.
211
+ #
212
+ # <tt>:keep_map_tasks</tt> BOOL or Fixnum (DEFAULT 1)
213
+ # If true, the master will run the map_tasks locally.
214
+ # If a number is provided, the master will run the map_tasks locally if there are
215
+ # LESS THAN OR EQUAL TO the number provided.
216
+ # You may also set Skynet::CONFIG[:DEFAILT_KEEP_MAP_TASKS] DEFAULT 1
217
+ #
218
+ # <tt>:keep_reduce_tasks</tt> BOOL or Fixnum (DEFAULT 1)
219
+ # If true, the master will run the reduce_tasks locally.
220
+ # If a number is provided, the master will run the reduce_tasks locally if there are
221
+ # LESS THAN OR EQUAL TO the number provided.
222
+ # You may also set Skynet::CONFIG[:DEFAILT_REDUCVE_MAP_TASKS] DEFAULT 1
223
+ def initialize(options = {})
224
+ FIELDS.each do |field|
225
+ if options.has_key?(field)
226
+ self.send("#{field}=".to_sym,options[field])
227
+ elsif Skynet::CONFIG[:JOB_DEFAULTS][field]
228
+ self.send("#{field}=".to_sym,Skynet::CONFIG[:JOB_DEFAULTS][field])
229
+ end
230
+ if options[:queue]
231
+ raise Error.new("The provided queue (#{options[:queue]}) does not exist in Skynet::CONFIG[:MESSAGE_QUEUES]") unless Skynet::CONFIG[:MESSAGE_QUEUES].index(options[:queue])
232
+ self.queue_id = Skynet::CONFIG[:MESSAGE_QUEUES].index(options[:queue])
233
+ end
234
+
235
+ # Backward compatability
236
+ self.mappers ||= options[:map_tasks]
237
+ self.reducers ||= options[:reduce_tasks]
238
+ end
239
+
240
+ raise Error.new("You can not run a local master in async mode.") if self.async and self.local_master
76
241
 
77
242
  @job_id = task_id
78
243
  end
79
244
 
80
- def to_h
81
- if @map.kind_of?(Proc) or @reduce.kind_of?(Proc)
82
- raise Skynet::Error.new("You have a Proc in your map or reduce. This can't be turned into a hash.")
245
+ # Options are:
246
+ # <tt>:local_master</tt> BOOL (DEFAULT true)
247
+ # By default, your Skynet::Job will act as the master for your map/reduce job, doling out
248
+ # tasks, waiting for other workers to complete and return their results and dealing with
249
+ # merging and partitioning the data. If you run in async mode, another worker will handle
250
+ # being the master for your job without blocking. If you run :async => false, :local_master => false
251
+ # Skynet will let another worker be the master for your job, but will block waiting for the
252
+ # final results. The benefit of this is that if your process dies, the Job will continue to
253
+ # run remotely.
254
+ #
255
+ # <tt>:async</tt> BOOL (DEFAULT false)
256
+ # If you run in async mode, another worker will handle being the master for your job without blocking.
257
+ # You can not pass :local_master => true, :async => true since the only way to allow your
258
+ # job to run asyncronously is to have a remote_master.
259
+ #
260
+ # You can pass any options you might pass to Skynet::Job.new. Warning: Passing options to run
261
+ # will permanently change properties of the job.
262
+ def run(options = {})
263
+ FIELDS.each do |field|
264
+ if options.has_key?(field)
265
+ self.send("#{field}=".to_sym,options[field])
266
+ end
83
267
  end
84
- hash = {}
85
- FIELDS.each do |field|
86
- next unless self.send(field)
87
- hash[field] = self.send(field)
268
+ raise Error.new("You can not run a local master in async mode.") if self.async and self.local_master
269
+
270
+ info "RUN 1 BEGIN #{name}, job_id:#{job_id} vers: #{version} async:#{async}, local_master: #{local_master}, master?: #{master?}"
271
+
272
+ # run the master task if we're running async or local_master
273
+ if master?
274
+ master_enqueue
275
+ # ====================================================================================
276
+ # = FIXME If async Return a handle to an object that can used to retrieve the results later.
277
+ # ====================================================================================
278
+ async? ? job_id : master_results
279
+ else
280
+ number_of_tasks_queued = self.map_enqueue
281
+ map_results = self.map_results(number_of_tasks_queued)
282
+ return map_results unless map_results and self.reduce
283
+
284
+ partitioned_data = self.partition_data(map_results)
285
+ return unless partitioned_data
286
+ number_of_tasks_queued = self.reduce_enqueue(partitioned_data)
287
+
288
+ @results = self.reduce_results(number_of_tasks_queued)
88
289
  end
89
- hash
90
290
  end
91
-
92
- def mq
93
- @mq ||= Skynet::MessageQueue.new
291
+
292
+ def master_enqueue
293
+ self.use_local_queue = local_master?
294
+ messages = tasks_to_messages([master_task])
295
+ enqueue_messages(messages)
94
296
  end
95
297
 
96
- ## set_version was supposed to know when to upgrade the version. Haven't figured out how to do this yet
97
- def set_version
98
- true
99
- # return 1 if solo?
100
- # oldver = mq.get_worker_version || 0
101
- # if oldver != self.version
102
- # mq.set_worker_version(self.version)
103
- # end
298
+ # Returns the final results of this map/reduce job. If results is called on an :async job
299
+ # calling results will block until results are found or the master_timeout is reached.
300
+ def results
301
+ # ============================================
302
+ # = FIXME Maybe this can have better warnings if the results aren't ready yet. =
303
+ # ============================================
304
+ master_results
104
305
  end
105
-
106
- def version
107
- return 1 if solo?
108
- @@worker_version ||= mq.get_worker_version
109
- @version ||= @@worker_version
306
+
307
+ def master_results
308
+ @results = gather_results(1,master_timeout,name) unless defined?(@results)
110
309
  end
111
310
 
112
- def version=(v)
113
- @version = v
311
+ def map_enqueue
312
+ task_ids = []
313
+ map_tasks = self.map_tasks
314
+ self.use_local_queue = map_local?
315
+ if map_tasks
316
+ number_of_tasks = 0
317
+ map_tasks.each do |task|
318
+ number_of_tasks += 1
319
+ enqueue_messages(tasks_to_messages(task))
320
+ end
321
+ end
322
+ return number_of_tasks
114
323
  end
115
324
 
116
- def display_info
117
- "#{name}, job_id: #{job_id}"
118
- end
119
-
120
- def increment_worker_version
121
- newver = mq.get_worker_version + 1
122
- mq.set_worker_version(newver)
123
- newver
124
- end
125
-
126
- def solo?
127
- (@solo or CONFIG[:SOLO])
128
- end
129
-
130
- def single?
131
- @single
325
+ def map_results(number_of_tasks)
326
+ debug "RUN MAP 2.4 BEFORE MAP #{display_info} MAP_LOCAL?:#{map_local?} USE_LOCAL_QUEUE?:#{use_local_queue?}"
327
+ results = gather_results(number_of_tasks, map_timeout, map_name)
328
+ return unless results
329
+ results.compact! if results.is_a?(Array)
330
+ debug "RUN MAP 2.5 RESULTS AFTER RUN #{display_info} MAP_LOCAL:#{map_local?} USE_LOCAL_QUEUE?:#{use_local_queue?} results:", results.inspect
331
+ results
132
332
  end
133
-
134
- def run_tasks(tasks,timeout = 5,description = "Generic Task")
135
- result = Hash.new
136
- errors = Hash.new
137
- mq = Skynet::MessageQueue.new unless solo?
138
- t1 = Time.now
139
- tasks = [tasks] unless tasks.class == Array
140
- info "RUN TASKS #{description} ver: #{self.version} jobid: #{job_id} @ #{t1}"
141
-
142
- # write tasks to the MessageQueue
143
- task_ids = []
144
- tasks.each do |task|
145
- debug "RUN TASKS SUBMITTING #{description} task #{task.task_id} job_id: #{job_id}"
146
- if solo? or single?
147
- result[task.task_id] = task.run
148
- else
149
- task_ids << task.task_id
150
- worker_message = Skynet::Message.new(
151
- :tasktype => :task,
152
- :job_id => job_id,
153
- :task_id => task.task_id,
154
- :payload => task,
155
- :payload_type => task.task_or_master,
156
- :expiry => timeout,
157
- :expire_time => @start_after,
158
- :iteration => 0,
159
- :name => description,
160
- :version => @version
161
- )
162
- debug "RUN TASKS WORKER MESSAGE #{description} job_id: #{job_id}", worker_message.to_a
163
- mq.write_message(worker_message,timeout * 5)
333
+
334
+ def partition_data(post_map_data)
335
+ debug "RUN REDUCE 3.1 BEFORE PARTITION #{display_info} reducers: #{reducers}"
336
+ debug "RUN REDUCE 3.1 : #{reducers} #{name}, job_id:#{job_id}", post_map_data
337
+ return unless post_map_data
338
+ partitioned_data = nil
339
+ if not @reduce_partition
340
+ # =====================
341
+ # = XXX HACK
342
+ # = There was a bug in Job where the reduce_partition of master jobs wasn't being set! This is to catch that.
343
+ # = It handles it by checking if the map class has a reduce partitioner. Maybe this is a good thing to leave anyway.
344
+ # =====================
345
+ if @map.is_a?(String) and @map.constantize.respond_to?(:reduce_partition)
346
+ partitioned_data = @map.constantize.reduce_partition(post_map_data, reducers)
347
+ else
348
+ partitioned_data = Skynet::Partitioners::RecombineAndSplit.reduce_partition(post_map_data, reducers)
164
349
  end
350
+ elsif @reduce_partition.is_a?(String)
351
+ partitioned_data = @reduce_partition.constantize.reduce_partition(post_map_data, reducers)
352
+ else
353
+ partitioned_data = @reduce_partition.call(post_map_data, reducers)
165
354
  end
355
+ partitioned_data.compact! if partitioned_data
356
+ debug "RUN REDUCE 3.2 AFTER PARTITION #{display_info} reducers: #{reducers}"
357
+ debug "RUN REDUCE 3.2 AFTER PARTITION #{display_info} data:", partitioned_data if partitioned_data
358
+ partitioned_data
359
+ end
166
360
 
167
- return result.values if solo? or single?
168
- return true if async
169
-
170
- debug "GATHER RESULTS for #{description} job_id: #{job_id} - NOT AN ASYNC JOB"
361
+ def reduce_enqueue(partitioned_data)
362
+ return partitioned_data unless @reduce and reducers and reducers > 0
363
+ debug "RUN REDUCE 3.3 CREATED REDUCE TASKS #{display_info}", partitioned_data
364
+
365
+ reduce_tasks = self.reduce_tasks(partitioned_data)
366
+ self.use_local_queue = reduce_local?(reduce_tasks)
367
+ number_of_tasks = 0
368
+ reduce_tasks.each do |task|
369
+ number_of_tasks += 1
370
+ enqueue_messages(tasks_to_messages(task))
371
+ end
372
+ return number_of_tasks
373
+ end
171
374
 
172
- # retrieve results unless async
375
+ def reduce_results(number_of_tasks)
376
+ results = gather_results(number_of_tasks, reduce_timeout, reduce_name)
377
+ if results.is_a?(Array) and results.first.is_a?(Hash)
378
+ hash_results = Hash.new
379
+ results.each {|h| hash_results.merge!(h) if h.class == Hash}
380
+ results = hash_results
381
+ elsif results.is_a?(Array) and results.first.is_a?(Array)
382
+ results = results.compact
383
+ end
384
+ debug "RUN REDUCE 3.4 AFTER REDUCE #{display_info} results size: #{results ? results.size : ''}"
385
+ debug "RUN REDUCE 3.4 AFTER REDUCE #{display_info} results:", results if results
386
+ return results
387
+ end
388
+
389
+ def enqueue_messages(messages)
390
+ messages.each do |message|
391
+ timeout = message.expiry || 5
392
+ debug "RUN TASKS SUBMITTING #{message.name} job_id: #{job_id} #{message.payload.is_a?(Skynet::Task) ? 'task' + message.payload.task_id.to_s : ''}"
393
+ debug "RUN TASKS WORKER MESSAGE #{message.name} job_id: #{job_id}", message.to_a
394
+ mq.write_message(message,timeout * 5)
395
+ end
396
+ end
397
+
398
+ def gather_results(number_of_tasks, timeout=nil, description=nil)
399
+ debug "GATHER RESULTS job_id: #{job_id} - NOT AN ASYNC JOB"
400
+ results = {}
401
+ errors = {}
402
+ started_at = Time.now.to_i
403
+
173
404
  begin
174
405
  loop do
175
406
  # debug "LOOKING FOR RESULT MESSAGE TEMPLATE"
176
- result_message = mq.take_result(job_id,timeout * 2)
407
+ result_message = self.mq.take_result(job_id,timeout * 2)
408
+ ret_result = result_message.payload
177
409
 
178
- ret_result = result_message.payload
179
410
  if result_message.payload_type == :error
180
411
  errors[result_message.task_id] = ret_result
181
412
  error "ERROR RESULT TASK #{result_message.task_id} returned #{errors[result_message.task_id].inspect}"
182
413
  else
183
- result[result_message.task_id] = ret_result
184
- debug "RESULT returned TASKID: #{result_message.task_id} #{result[result_message.task_id].inspect}"
414
+ results[result_message.task_id] = ret_result
415
+ debug "RESULT returned TASKID: #{result_message.task_id} #{results[result_message.task_id].inspect}"
185
416
  end
186
- debug "RESULT collected: #{(result.keys + errors.keys).size}, remaining: #{(task_ids - (result.keys + errors.keys)).size}"
187
- break if (task_ids - (result.keys + errors.keys)).empty?
188
- if (task_ids - (result.keys & errors.keys)).empty?
189
- raise Skynet::Job::Error.new("WORKER ERROR #{description}, job_id: #{job_id} errors:#{errors.keys.size} out of #{task_ids.size} workers. #{errors.pretty_print_inspect}")
190
- end
417
+ debug "RESULT collected: #{(results.keys + errors.keys).size}, remaining: #{(number_of_tasks - (results.keys + errors.keys).uniq.size)}"
418
+ break if (number_of_tasks - (results.keys + errors.keys).uniq.size) <= 0
191
419
  end
192
- rescue Skynet::RequestExpiredError => e
420
+ rescue Skynet::RequestExpiredError => e
421
+ local_mq_reset!
193
422
  error "A WORKER EXPIRED or ERRORED, #{description}, job_id: #{job_id}"
194
423
  if not errors.empty?
195
- raise WorkerError.new("WORKER ERROR #{description}, job_id: #{job_id} errors:#{errors.keys.size} out of #{task_ids.size} workers. #{errors.pretty_print_inspect}")
424
+ raise WorkerError.new("WORKER ERROR #{description}, job_id: #{job_id} errors:#{errors.keys.size} out of #{number_of_tasks} workers. #{errors.pretty_print_inspect}")
196
425
  else
197
426
  raise Skynet::RequestExpiredError.new("WORKER ERROR, A WORKER EXPIRED! Did not get results or even errors back from all workers!")
198
427
  end
199
428
  end
200
- info "RUN TASKS COMPLETE #{description} jobid: #{job_id} TOOK: #{Time.now - t1}"
201
- result.values
202
- end
203
-
204
- def map_reduce_class=(klass)
205
- unless klass.class == String or klass.class == Class
206
- raise BadMapOrReduceError.new("#{self.class}.map_reduce only accepts a class name")
207
- end
208
- klass = klass.to_s
209
- @map = klass
210
- self.name ||= "#{klass} MASTER"
211
- self.map_name ||= "#{klass} MAP"
212
- if klass.constantize.respond_to?(:reduce)
213
- @reduce = klass
214
- self.reduce_name ||= "#{klass} REDUCE"
429
+ local_mq_reset!
430
+
431
+ # ==========
432
+ # = FIXME Tricky one. Should we throw an exception if we didn't get all the results back, or should we keep going.
433
+ # = Maybe this is another needed option.
434
+ # ==========
435
+ # if not (errors.keys - results.keys).empty?
436
+ # raise WorkerError.new("WORKER ERROR #{description}, job_id: #{job_id} errors:#{errors.keys.size} out of #{number_of_tasks} workers. #{errors.pretty_print_inspect}")
437
+ # end
438
+
439
+ return nil if results.values.compact.empty?
440
+ return results.values
441
+ end
442
+
443
+ def local_mq_reset!
444
+ if use_local_queue?
445
+ local_mq.reset!
446
+ self.use_local_queue=false
215
447
  end
216
- @reduce_partitioner = klass if klass.constantize.respond_to?(:reduce_partitioner)
217
- @map_partitioner = klass if klass.constantize.respond_to?(:map_partitioner)
218
448
  end
449
+
219
450
 
220
- def task_id
221
- @task_id ||= get_unique_id(1)
222
- end
223
-
224
- # def run_master
225
- # result = run_tasks(master_task,master_timeout,name)
226
- # debug "MASTER RESULT #{self.name} job_id: #{self.job_id}", result
227
- # result
228
- # end
229
-
230
451
  def master_task
231
452
  @master_task ||= begin
232
453
  raise Exception.new("No map provided") unless @map
233
- set_version
234
-
235
- job = Skynet::Job.new(
236
- :map_timeout => map_timeout,
237
- :reduce_timeout => reduce_timeout,
238
- :job_id => :task_id,
239
- :map_data => @map_data,
240
- :map_name => map_name || name,
241
- :reduce_name => reduce_name || name,
242
- :map => @map,
243
- :map_partitioner => @map_partitioner,
244
- :reduce => @reduce,
245
- :reduce_partitioner => @reduce_partitioner,
246
- :map_tasks => @map_tasks,
247
- :reduce_tasks => @reduce_tasks,
248
- :name => @name,
249
- :version => version,
250
- :process => lambda do |data|
251
- debug "RUNNING MASTER RUN #{name}, job_id:#{job_id}"
252
- job.run
454
+
455
+ # Make sure to set single to false in our own Job object.
456
+ # We're just passing along whether they set us to single.
457
+ # If we were single, we'd never send off the master to be run externally.
458
+ @single = false
459
+
460
+ task = Skynet::Task.master_task(self)
461
+ end
462
+ end
463
+
464
+ def map_tasks
465
+ @map_tasks ||= begin
466
+ map_tasks = []
467
+ debug "RUN MAP 2.1 #{display_info} data size before partition: #{@map_data.size}" if @map_data.respond_to?(:size)
468
+ debug "RUN MAP 2.1 #{display_info} data before partition:", @map_data
469
+
470
+ task_options = {
471
+ :process => @map,
472
+ :name => map_name,
473
+ :map_or_reduce => :map,
474
+ :result_timeout => map_timeout,
475
+ :retry => map_retry || Skynet::CONFIG[:DEFAULT_MAP_RETRY]
476
+ }
477
+
478
+ if @map_data.is_a?(Array)
479
+ debug "RUN MAP 2.2 DATA IS Array #{display_info}"
480
+ num_mappers = @map_data.length < @mappers ? @map_data.length : @mappers
481
+
482
+ map_data = if @map_partitioner
483
+ @map_partitioner.call(@map_data,num_mappers)
484
+ else
485
+ Skynet::Partitioners::SimplePartitionData.reduce_partition(@map_data, num_mappers)
253
486
  end
254
- )
255
-
256
- task = Skynet::Task.new(
257
- :task_id => task_id,
258
- :data => :master,
259
- :process => process,
260
- :map_or_reduce => :master,
261
- :name => self.name,
262
- :result_timeout => master_result_timeout
263
- )
487
+
488
+ debug "RUN MAP 2.3 #{display_info} data size after partition: #{map_data.size}"
489
+ debug "RUN MAP 2.3 #{display_info} map data after partition:", map_data
490
+ elsif @map_data.is_a?(Enumerable)
491
+ debug "RUN MAP 2.2 DATA IS ENUMERABLE #{display_info} map_data_class: #{@map_data.class}"
492
+ map_data = @map_data
493
+ else
494
+ debug "RUN MAP 2.2 DATA IS NOT ARRAY OR ENUMERABLE #{display_info} map_data_class: #{@map_data.class}"
495
+ map_data = [ @map_data ]
496
+ end
497
+ Skynet::TaskIterator.new(task_options, map_data)
264
498
  end
265
499
  end
266
-
267
- # Run the job and return result arrays
268
- def run
269
- run_job
500
+
501
+ def reduce_tasks(partitioned_data)
502
+ @reduce_tasks ||= begin
503
+ task_options = {
504
+ :name => reduce_name,
505
+ :process => @reduce,
506
+ :map_or_reduce => :reduce,
507
+ :result_timeout => reduce_timeout,
508
+ :retry => reduce_retry || Skynet::CONFIG[:DEFAULT_REDUCE_RETRY]
509
+ }
510
+ Skynet::TaskIterator.new(task_options, partitioned_data)
511
+ end
270
512
  end
271
513
 
272
- def run_job
273
- debug "RUN 1 BEGIN #{name}, job_id:#{job_id}"
274
- set_version
275
- # unless (@map && @reduce)
276
- raise ArgumentError, "map lambdas not assigned" unless (@map)
277
-
278
- # sometimes people want to run a master with just run. In this case we assume we have to set the data to the map_data
279
- # XXX seems like a hack
514
+ def tasks_to_messages(tasks)
515
+ if tasks.is_a?(Skynet::TaskIterator)
516
+ tasks = tasks.to_a
517
+ elsif not tasks.is_a?(Array)
518
+ tasks = [tasks]
519
+ end
280
520
 
281
- debug "RUN 2 MAP pre run_map #{name}, job_id:#{job_id}"
521
+ tasks.collect do |task|
522
+ Skynet::Message.new_task_message(task,self)
523
+ end
524
+ end
282
525
 
283
- post_map_data = run_map
284
- debug "RUN 3 REDUCE pre run_reduce #{name}, job_id:#{job_id}"
285
- return post_map_data unless post_map_data
286
- results = run_reduce(post_map_data)
287
- debug "RUN 4 FINISHED run_job #{name}, job_id:#{job_id}"
288
- results
526
+ def master_retry
527
+ @master_retry || Skynet::CONFIG[:DEFAULT_MASTER_RETRY]
528
+ end
529
+
530
+ def map_retry
531
+ @map_retry || Skynet::CONFIG[:DEFAULT_MAP_RETRY]
532
+ end
533
+
534
+ def reduce_retry
535
+ @reduce_retry || Skynet::CONFIG[:DEFAULT_REDUCE_RETRY]
289
536
  end
290
-
291
- # Partition up starting data, create map tasks
292
- def run_map
293
- map_tasks = Array.new
294
- debug "RUN MAP 2.1 #{display_info} data size before partition: #{@map_data.size}"
295
- debug "RUN MAP 2.1 #{display_info} data before partition:", @map_data
296
- if @map_data.class == Array
297
- debug "RUN MAP 2.2 DATA IS Array #{display_info}"
298
- num_mappers = @map_data.length < @map_tasks ? @map_data.length : @map_tasks
299
- pre_map_data = Array.new
300
- if @map_partitioner
301
- pre_map_data = @map_partitioner.call(@map_data,num_mappers)
302
- else
303
- pre_map_data = Partitioner::simple_partition_data(@map_data, num_mappers)
304
- end
305
- debug "RUN MAP 2.3 #{display_info} data size after partition: #{pre_map_data.size}"
306
- debug "RUN MAP 2.3 #{display_info} map data after partition:", pre_map_data
307
- map_tasks = Array.new
308
-
309
- (0..num_mappers - 1).each do |i|
310
- map_tasks << Skynet::Task.new(
311
- :task_id => get_unique_id(1),
312
- :data => pre_map_data[i],
313
- :process => @map,
314
- :name => map_name,
315
- :map_or_reduce => :map,
316
- :result_timeout => result_timeout
317
- )
318
- end
319
-
320
- # Run map tasks
321
- #
322
- elsif @map_data.is_a?(Enumerable)
323
- debug "RUN MAP 2.2 DATA IS ENUMERABLE #{display_info} map_data_class: #{@map_data.class}"
324
- each_method = @map_data.respond_to?(:next) ? :next : :each
325
- @map_data.send(each_method) do |pre_map_data|
326
- map_tasks << Skynet::Task.new(
327
- :task_id => get_unique_id(1),
328
- :data => pre_map_data,
329
- :process => @map,
330
- :name => map_name,
331
- :map_or_reduce => :map,
332
- :result_timeout => result_timeout
333
- )
334
- end
335
- else
336
- debug "RUN MAP 2.2 DATA IS NOT ARRAY OR ENUMERABLE #{display_info} map_data_class: #{@map_data.class}"
337
- map_tasks = [
338
- Skynet::Task.new(
339
- :task_id => get_unique_id(1),
340
- :data => @map_data,
341
- :process => @map,
342
- :name => map_name,
343
- :map_or_reduce => :map,
344
- :result_timeout => result_timeout
345
- )
346
- ]
347
- end
348
537
 
349
- begin
350
- post_map_data = run_tasks(map_tasks,map_timeout,map_name)
351
- rescue WorkerError => e
352
- error "MAP FAILED #{display_info} #{e.class} #{e.message.inspect}"
353
- return nil
354
- end
538
+ def keep_map_tasks
539
+ @keep_map_tasks || Skynet::CONFIG[:DEFAULT_KEEP_MAP_TASKS]
540
+ end
355
541
 
356
- debug "RUN MAP 2.5 RESULTS AFTER RUN #{display_info} results:", post_map_data.inspect
542
+ def keep_reduce_tasks
543
+ @keep_reduce_tasks || Skynet::CONFIG[:DEFAULT_KEEP_REDUCE_TASKS]
544
+ end
545
+
546
+ def map_local?
547
+ return true if solo? or single?
548
+ return true if keep_map_tasks == true
549
+ # error "RUN MAP 2.4 BEFORE MAP #{display_info} KEEPMT:#{keep_map_tasks} DKMT:#{Skynet::CONFIG[:DEFAULT_KEEP_MAP_TASKS]} MDCLASS: #{map_tasks.data.class} #{(map_tasks.data.is_a?(Array) ? map_tasks.data.size : '')}"
550
+ return true if keep_map_tasks and map_tasks.data.is_a?(Array) and map_tasks.data.size <= keep_map_tasks
551
+ return false
552
+ end
357
553
 
358
- return nil unless post_map_data
554
+ def reduce_local?(reduce_tasks)
555
+ return true if solo? or single?
556
+ return true if keep_reduce_tasks == true
557
+ return true if keep_reduce_tasks and reduce_tasks.data.is_a?(Array) and reduce_tasks.data.size <= keep_reduce_tasks
558
+ return false
559
+ end
359
560
 
360
- post_map_data.compact! if post_map_data.class == Array
361
-
362
- return post_map_data
561
+ def use_local_queue?
562
+ @use_local_queue
363
563
  end
364
-
365
- # Re-partition returning data for reduction, create reduce tasks
366
- def run_reduce(post_map_data=nil)
367
- return post_map_data unless post_map_data and @reduce
368
564
 
369
- num_reducers = @reduce_tasks
565
+ # async is true if the async flag is set and the job is not a 'single' job, or in solo mode.
566
+ # async only applies to whether we run the master locally and whether we poll for the result
567
+ def async?
568
+ @async and not (solo? or single? or local_master?)
569
+ end
570
+
571
+ def master?
572
+ async? or not local_master?
573
+ end
370
574
 
371
- debug "RUN REDUCE 3.1 BEFORE PARTITION #{display_info} num_reducers: #{num_reducers}"
372
- # debug "RUN REDUCE 3.1 : #{num_reducers} #{name}, job_id:#{job_id}", post_map_data
373
-
374
- reduce_data = run_reduce_partitioner(post_map_data, num_reducers)
375
- reduce_data.compact!
376
- debug "RUN REDUCE 3.2 AFTER PARTITION #{display_info} num_reducers: #{reduce_data.length}"
377
- debug "RUN REDUCE 3.2 AFTER PARTITION #{display_info} data:", reduce_data
378
- reduce_tasks = Array.new
379
-
380
- (0..reduce_data.length - 1).each do |i|
381
- reduce_tasks << Skynet::Task.new(
382
- :task_id => get_unique_id(1),
383
- :data => reduce_data[i],
384
- :name => reduce_name,
385
- :process => @reduce,
386
- :map_or_reduce => :reduce,
387
- :result_timeout => result_timeout
388
- )
389
- end
390
- reduce_tasks.compact! if reduce_tasks
391
-
392
- debug "RUN REDUCE 3.3 CREATED REDUCE TASKS #{display_info}"#, reduce_tasks
393
-
394
- # Reduce and return results
395
- #
396
- begin
397
- results = run_tasks(reduce_tasks, reduce_timeout,reduce_name)
398
- rescue WorkerError => e
399
- error "REDUCE FAILED #{display_info} #{e.class} #{e.message.inspect}"
400
- return nil
401
- end
575
+ def local_master?
576
+ @local_master or solo?
577
+ end
402
578
 
403
- if results.class == Array and results.first.class == Hash
404
- hash_results = Hash.new
405
- results.each {|h| hash_results.merge!(h) if h.class == Hash}
406
- # results.flatten! if results
407
- results = hash_results
579
+ def solo?
580
+ (@solo or CONFIG[:SOLO])
581
+ end
582
+
583
+ def single?
584
+ @single
585
+ end
586
+
587
+ def reset!
588
+ @map_tasks = nil
589
+ @reduce_tasks = nil
590
+ end
591
+
592
+ def to_h
593
+ if @map.kind_of?(Proc) or @reduce.kind_of?(Proc)
594
+ raise Skynet::Error.new("You have a Proc in your map or reduce. This can't be turned into a hash.")
408
595
  end
409
- debug "RUN REDUCE 3.4 AFTER REDUCE #{display_info} results size: #{results.size}"
410
- debug "RUN REDUCE 3.4 AFTER REDUCE #{display_info} results:", results
411
- return results
596
+ hash = {}
597
+ FIELDS.each do |field|
598
+ hash[field] = self.send(field) if self.send(field)
599
+ end
600
+ hash
412
601
  end
413
-
414
- def run_reduce_partitioner(post_map_data,num_reducers)
415
- if not @reduce_partitioner
416
- Partitioner::recombine_and_split.call(post_map_data, num_reducers)
417
- elsif @reduce_partitioner.class == String
418
- @reduce_partitioner.constantize.reduce_partitioner(post_map_data, num_reducers)
419
- else
420
- @reduce_partitioner.call(post_map_data, num_reducers)
602
+
603
+ def task_id
604
+ @task_id ||= get_unique_id(1).to_i
605
+ end
606
+
607
+ def version
608
+ return 1 if solo?
609
+ @version ||= begin
610
+ @@worker_version ||= self.mq.get_worker_version || 1
611
+ @@worker_version
421
612
  end
422
613
  end
423
-
424
- end ### END class Skynet::Job
425
-
426
- class AsyncJob < Skynet::Job
427
-
428
- ## XXX Partitioning doesn't work yet!!!!!
429
614
 
430
- def initialize(opts = {})
431
- opts[:async] = true
432
- super(opts)
615
+ def version=(v)
616
+ @version = v
433
617
  end
618
+
619
+ def display_info
620
+ "#{name}, job_id: #{job_id}"
621
+ end
434
622
 
435
-
436
- def map=(klass)
437
- unless klass.class == String or klass.class == Class
438
- raise BadMapOrReduceError.new("#{self.class}.map only accepts a class name")
439
- end
440
- klass = klass.to_s if klass.class == Symbol
441
- @map = klass
623
+ def start_after=(time)
624
+ @start_after = (time.is_a?(Time) ? time.to_i : time)
442
625
  end
626
+
627
+ def map_data=(map_data)
628
+ reset!
629
+ @map_data = map_data
630
+ end
631
+
632
+ def map=(map)
633
+ reset!
634
+ return unless map
635
+ if map.class == String or map.class == Class
636
+ @map = map.to_s
637
+ elsif map.is_a?(Proc)
638
+ @map = map
639
+ else
640
+ raise BadMapOrReduceError.new("#{self.class}.map accepts a class name or a proc. Got #{map}")
641
+ end
642
+ end
643
+
644
+ def reduce=(reduce)
645
+ reset!
646
+ return unless reduce
647
+ if reduce.class == String or reduce.class == Class
648
+ @reduce = reduce.to_s
649
+ elsif reduce.is_a?(Proc)
650
+ @reduce = reduce
651
+ else
652
+ raise BadMapOrReduceError.new("#{self.class}.reduce accepts a class name or a proc. Got #{reduce}")
653
+ end
654
+ end
443
655
 
444
- def reduce=(klass)
656
+ def map_reduce_class=(klass)
657
+ reset!
445
658
  unless klass.class == String or klass.class == Class
446
- raise BadMapOrReduceError.new("#{self.class}.reduce only accepts a class name")
659
+ raise BadMapOrReduceError.new("#{self.class}.map_reduce only accepts a class name: #{klass} #{klass.class}")
447
660
  end
448
- klass = klass.to_s if klass.class == Symbol
449
- @reduce = klass
450
- end
451
-
452
- def run_master
453
- if solo?
454
- run_job
455
- else
456
- results = run_tasks(master_task,master_timeout,name)
457
- self.job_id
661
+ klass = klass.to_s
662
+ @map = klass
663
+ self.name ||= "#{klass} MASTER"
664
+ self.map_name ||= "#{klass} MAP"
665
+ if klass.constantize.respond_to?(:reduce)
666
+ @reduce ||= klass
667
+ self.reduce_name ||= "#{klass} REDUCE"
458
668
  end
669
+ @reduce_partitioner ||= klass if klass.constantize.respond_to?(:reduce_partition)
670
+ @map_partitioner ||= klass if klass.constantize.respond_to?(:map_partitioner)
459
671
  end
460
672
 
461
- def master_task
462
- @master_task ||= begin
463
- raise Exception.new("No map provided") unless @map
464
- set_version
465
- job = Skynet::Job.new(
466
- :map_timeout => map_timeout,
467
- :reduce_timeout => reduce_timeout,
468
- :job_id => task_id,
469
- :map_data => @map_data,
470
- :map_name => map_name || name,
471
- :reduce_name => reduce_name || name,
472
- :map => @map,
473
- :map_partitioner => @map_partitioner,
474
- :reduce => @reduce,
475
- :reduce_partitioner => @reduce_partitioner,
476
- :map_tasks => @map_tasks,
477
- :reduce_tasks => @reduce_tasks,
478
- :name => @name,
479
- :version => version,
480
- :result_timeout => result_timeout,
481
- :master_result_timeout => master_result_timeout,
482
- :solo => solo,
483
- :single => single
484
- )
485
- @single = false
486
-
487
- task = Skynet::Task.new(
488
- :task_id => task_id,
489
- :data => nil,
490
- :process => job.to_h,
491
- :map_or_reduce => :master,
492
- :name => self.name,
493
- :result_timeout => master_result_timeout
494
- )
495
- end
673
+ def run_master
674
+ error "run_master has been deprecated, please use run"
675
+ run(:local_master => false)
496
676
  end
497
677
 
498
- def run
499
- if solo?
500
- super
678
+ def mq
679
+ if use_local_queue?
680
+ local_mq
501
681
  else
502
- run_master
682
+ @mq ||= Skynet::MessageQueue.new
503
683
  end
504
684
  end
505
-
506
- end ### END class Skynet::AsyncJob
685
+
686
+ def local_mq
687
+ @local_mq ||= LocalMessageQueue.new
688
+ end
507
689
 
508
- # Collection of partitioning utilities
509
- #
510
- module Partitioner
690
+ def self.mq
691
+ Skynet::MessageQueue.new
692
+ end
693
+
694
+ end ### END class Skynet::Job
695
+ end
696
+
697
+ class Skynet::AsyncJob < Skynet::Job
698
+ # Skynet::AsyncJob is for Skynet jobs you want to run asyncronously.
699
+ # Normally when you run a Skynet::Job it blocks until the job is complete.
700
+ # Running an Async job merely returns a job_id which can be used later to retrieve the results.
701
+ # See Skynet::Job for full documentation
511
702
 
512
- # Split one block of data into partitions
513
- #
514
- def self.args_pp(*args)
515
- "#{args.length > 0 ? args.pretty_print_inspect : ''}"
703
+ def initialize(options = {})
704
+ options[:async] = true
705
+ options[:local_master] = false
706
+ super(options)
707
+ end
708
+
709
+ def map=(klass)
710
+ unless klass.class == String or klass.class == Class
711
+ raise BadMapOrReduceError.new("#{self.class}.map only accepts a class name")
516
712
  end
713
+ @map = klass.to_s
714
+ end
517
715
 
518
- def self.debug(msg,*args)
519
- log = Skynet::Logger.get
520
- log.debug "#{self.class} PARTITION: #{msg} #{args_pp(*args)}"
716
+ def reduce=(klass)
717
+ unless klass.class == String or klass.class == Class
718
+ raise BadMapOrReduceError.new("#{self.class}.reduce only accepts a class name")
521
719
  end
720
+ @reduce = klass.to_s
721
+ end
722
+ end # class Skynet::AsyncJob
522
723
 
523
- def self.simple_partition_data(data, partitions)
524
- partitioned_data = Array.new
724
+ class Skynet::Job::LocalMessageQueue
725
+ include SkynetDebugger
726
+
727
+ attr_reader :messages, :results
525
728
 
526
- # If data size is significantly greater than the number of desired
527
- # partitions, we can divide the data roughly but the last partition
528
- # may be smaller than the others.
529
- #
530
- return data if (not data) or data.empty?
531
-
532
- if partitions >= data.length
533
- data.each do |datum|
534
- partitioned_data << [datum]
535
- end
536
- elsif (data.length >= partitions * 2)
537
- # Use quicker but less "fair" method
538
- size = data.length / partitions
729
+ def initialize
730
+ @messages = []
731
+ @results = []
732
+ end
733
+
734
+ def get_worker_version
735
+ 1
736
+ end
539
737
 
540
- if (data.length % partitions != 0)
541
- size += 1 # Last slice of leftovers
542
- end
738
+ def take_result(job_id,timeout=nil)
739
+ raise Skynet::RequestExpiredError.new if @messages.empty?
740
+ run_message(@messages.shift)
741
+ end
543
742
 
544
- (0..partitions - 1).each do |i|
545
- partitioned_data[i] = data[i * size, size]
546
- end
547
- else
548
- # Slower method, but partitions evenly
549
- partitions = (data.size < partitions ? data.size : partitions)
550
- (0..partitions - 1).each { |i| partitioned_data[i] = Array.new }
551
-
552
- data.each_with_index do |datum, i|
553
- partitioned_data[i % partitions] << datum
554
- end
555
- end
556
-
557
- partitioned_data
558
- end
743
+ def write_message(message,timeout=nil)
744
+ @messages << message
745
+ end
559
746
 
560
- # Tries to be smart about what kind of data its getting, whether array of arrays or array of arrays of arrays.
561
- #
562
- def self.recombine_and_split
563
- lambda do |post_map_data, new_partitions|
747
+ def empty?
748
+ @messages.empty?
749
+ end
564
750
 
565
- return post_map_data unless post_map_data.is_a?(Array) and (not post_map_data.empty?) and post_map_data.first.is_a?(Array) and (not post_map_data.first.empty?)
566
- if not post_map_data.first.first.is_a?(Array)
567
- partitioned_data = post_map_data.flatten
568
- else
569
- partitioned_data = post_map_data.inject(Array.new) do |data,part|
570
- data += part
571
- end
572
- end
573
- partitioned_data = Partitioner::simple_partition_data(partitioned_data, new_partitions)
574
- debug "POST PARTITIONED DATA", partitioned_data
575
- partitioned_data
751
+ def in_use?
752
+ (not empty?)
753
+ end
754
+
755
+ def reset!
756
+ @messages = []
757
+ @results = []
758
+ end
759
+
760
+ def run_message(message)
761
+ result = nil
762
+ (message.retry + 1).times do
763
+ task = message.payload
764
+ debug "RUN TASKS LOCALLY SUBMITTING #{message.name} task #{task.task_id}", task
765
+ begin
766
+ result = task.run
767
+ break
768
+ rescue Skynet::Task::TimeoutError => e
769
+ result = e
770
+ error "Skynet::Job::LocalMessageQueue Task timed out while executing #{e.inspect} #{e.backtrace.join("\n")}"
771
+ next
772
+ rescue Exception => e
773
+ error "Skynet::Job::LocalMessageQueue :#{__LINE__} #{e.inspect} #{e.backtrace.join("\n")}"
774
+ result = e
775
+ next
576
776
  end
577
777
  end
778
+ message.result_message(result)
779
+ end
780
+ end # class LocalMessageQueue
781
+
782
+ class Skynet::TaskIterator
783
+ include SkynetDebugger
784
+ include Skynet::GuidGenerator
578
785
 
786
+ class Error < StandardError
787
+ end
579
788
 
580
- # Smarter partitioner for array data, generates simple sum of array[0]
581
- # and ensures that all arrays sharing that key go into the same partition.
582
- #
583
- def self.array_data_split_by_first_entry
584
- lambda do |partitioned_data, new_partitions|
585
- partitions = Array.new
586
- (0..new_partitions - 1).each { |i| partitions[i] = Array.new }
587
-
588
- partitioned_data.each do |partition|
589
- partition.each do |array|
590
- next unless array.class == Array and array.size == 2
591
- if array[0].kind_of?(Fixnum)
592
- key = array[0]
593
- else
594
- key = 0
595
- array[0].each_byte { |c| key += c }
596
- end
597
- partitions[key % new_partitions] << array
598
- end
599
- end
789
+ include Enumerable
600
790
 
601
- partitions
602
- end
791
+ attr_accessor :task_options, :data
792
+
793
+ def initialize(task_options, data)
794
+ @task_options = task_options
795
+ @data = data
796
+ end
797
+
798
+ def first
799
+ if data.respond_to?(:first)
800
+ @first ||= Skynet::Task.new(task_options.merge(:data => data.first, :task_id => get_unique_id(1).to_i))
801
+ else
802
+ raise Error.new("#{data.class} does not implement 'first'")
603
803
  end
804
+ end
805
+
806
+ def size
807
+ if data.respond_to?(:size)
808
+ data.size
809
+ else
810
+ raise Error.new("#{data.class} does not implement 'size'")
811
+ end
812
+ end
604
813
 
814
+ def [](index)
815
+ if data.respond_to?(:[])
816
+ Skynet::Task.new(task_options.merge(:data => data[index], :task_id => get_unique_id(1).to_i))
817
+ else
818
+ raise Error.new("#{data.class} does not implement '[]'")
819
+ end
820
+ end
821
+
822
+ def each_method
823
+ each_method = data.respond_to?(:next) ? :next : :each
824
+ end
825
+
826
+ def to_a
827
+ self.collect { |task| task }
605
828
  end
606
829
 
607
- end
830
+ def each
831
+ iteration = 0
832
+ data.send(each_method) do |task_data|
833
+ task = nil
834
+ if @first and iteration == 0
835
+ task = @first
836
+ else
837
+ task = Skynet::Task.new(task_options.merge(:data => task_data, :task_id => (get_unique_id(1).to_i)))
838
+ @first = task if iteration == 0
839
+ end
840
+ iteration += 1
841
+ yield task
842
+ end
843
+ end
844
+ end # class TaskIterator
845
+
846
+
847
+ # require 'ruby2ruby' # XXX this will break unless people have the fix to Ruby2Ruby
848
+ ##### ruby2ruby fix from ruby2ruby.rb ############
849
+ ### XXX This is bad. Some people rely on an exception being thrown if a method is missing! BULLSHIT!
850
+ # class NilClass # Objective-C trick
851
+ # def method_missing(msg, *args, &block)
852
+ # nil
853
+ # end
854
+ # end
855
+ ##############################