pwrake 2.0.1 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +22 -9
- data/bin/gfwhere-pipe +33 -9
- data/bin/pwrake +5 -2
- data/bin/pwrake_branch +5 -3
- data/lib/pwrake/branch/branch.rb +95 -86
- data/lib/pwrake/branch/branch_application.rb +4 -0
- data/lib/pwrake/branch/communicator.rb +173 -0
- data/lib/pwrake/branch/communicator_set.rb +100 -0
- data/lib/pwrake/branch/fiber_queue.rb +10 -0
- data/lib/pwrake/branch/shell.rb +68 -24
- data/lib/pwrake/branch/shell_profiler.rb +2 -0
- data/lib/pwrake/gfarm/gfarm_postprocess.rb +8 -7
- data/lib/pwrake/logger.rb +5 -0
- data/lib/pwrake/master/master.rb +190 -87
- data/lib/pwrake/master/master_application.rb +8 -0
- data/lib/pwrake/nbio.rb +525 -0
- data/lib/pwrake/option/host_map.rb +36 -4
- data/lib/pwrake/option/option.rb +7 -1
- data/lib/pwrake/option/option_filesystem.rb +13 -3
- data/lib/pwrake/queue/locality_aware_queue.rb +41 -6
- data/lib/pwrake/queue/queue_array.rb +31 -11
- data/lib/pwrake/queue/task_queue.rb +15 -18
- data/lib/pwrake/report/report.rb +2 -0
- data/lib/pwrake/task/task_algorithm.rb +4 -1
- data/lib/pwrake/task/task_manager.rb +2 -0
- data/lib/pwrake/task/task_property.rb +1 -0
- data/lib/pwrake/task/task_wrapper.rb +40 -21
- data/lib/pwrake/version.rb +1 -1
- data/lib/pwrake/worker/invoker.rb +4 -29
- data/pwrake.gemspec +3 -2
- metadata +24 -12
- data/lib/pwrake/branch.rb +0 -22
- data/lib/pwrake/branch/worker_communicator.rb +0 -104
- data/lib/pwrake/iomux/channel.rb +0 -70
- data/lib/pwrake/iomux/handler.rb +0 -124
- data/lib/pwrake/iomux/handler_set.rb +0 -35
- data/lib/pwrake/iomux/runner.rb +0 -62
- data/lib/pwrake/master.rb +0 -30
@@ -1,35 +0,0 @@
|
|
1
|
-
module Pwrake
|
2
|
-
|
3
|
-
class HandlerSet < Array
|
4
|
-
|
5
|
-
def kill(sig)
|
6
|
-
each do |hdl|
|
7
|
-
hdl.iow.puts("kill:#{sig}")
|
8
|
-
hdl.iow.flush
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
def exit
|
13
|
-
a = []
|
14
|
-
each do |hdl|
|
15
|
-
iow = hdl.iow
|
16
|
-
begin
|
17
|
-
iow.puts "exit"
|
18
|
-
iow.flush
|
19
|
-
a << hdl
|
20
|
-
Log.debug "HandlerSet#exit iow=#{iow.inspect}"
|
21
|
-
rescue Errno::EPIPE => e
|
22
|
-
if Rake.application.options.debug
|
23
|
-
$stderr.puts "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}"
|
24
|
-
$stderr.puts e.backtrace.join("\n")
|
25
|
-
end
|
26
|
-
Log.error "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}\n"+
|
27
|
-
e.backtrace.join("\n")
|
28
|
-
end
|
29
|
-
end
|
30
|
-
a.each{|hdl| hdl.wait_message("exited")}
|
31
|
-
Log.debug "HandlerSet#exit end"
|
32
|
-
end
|
33
|
-
|
34
|
-
end
|
35
|
-
end
|
data/lib/pwrake/iomux/runner.rb
DELETED
@@ -1,62 +0,0 @@
|
|
1
|
-
module Pwrake
|
2
|
-
|
3
|
-
class TimeoutError < IOError; end
|
4
|
-
|
5
|
-
class Runner
|
6
|
-
|
7
|
-
def initialize(timeout=nil)
|
8
|
-
@timeout = timeout
|
9
|
-
@handler = {}
|
10
|
-
@hb_time = {}
|
11
|
-
end
|
12
|
-
|
13
|
-
attr_reader :handler
|
14
|
-
|
15
|
-
def add_handler(hdl)
|
16
|
-
@handler[hdl.ior] ||= hdl
|
17
|
-
end
|
18
|
-
|
19
|
-
def delete_handler(hdl)
|
20
|
-
@handler.delete(hdl.ior)
|
21
|
-
end
|
22
|
-
|
23
|
-
def run
|
24
|
-
while !(io_set = @handler.keys).empty?
|
25
|
-
sel, = IO.select(io_set,nil,nil,@timeout)
|
26
|
-
if sel.nil?
|
27
|
-
raise TimeoutError,"Timeout (#{@timeout} s) in IO.select"
|
28
|
-
end
|
29
|
-
sel.each do |io|
|
30
|
-
@handler[io].process_line
|
31
|
-
end
|
32
|
-
if @timeout && @hb_earliest
|
33
|
-
if Time.now - @hb_earliest > @timeout
|
34
|
-
io = @hb_time.key(@hb_earliest)
|
35
|
-
raise TimeoutError,"Timeout (#{@timeout}s) "+
|
36
|
-
"in Heartbeat from host=#{get_host(io)}"
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
def finish
|
43
|
-
@handler.each do |io,hdl|
|
44
|
-
hdl.finish
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
# used to print an error message
|
49
|
-
def get_host(io)
|
50
|
-
hdl = @handler[io]
|
51
|
-
h = hdl.respond_to?(:host) ? hdl.host : nil
|
52
|
-
end
|
53
|
-
|
54
|
-
# called when IO start and receive heartbeat
|
55
|
-
def heartbeat(io)
|
56
|
-
@hb_time[io] = Time.now
|
57
|
-
@hb_earliest = @hb_time.values.min
|
58
|
-
#Log.debug "heartbeat: host=#{get_host(io)}"
|
59
|
-
end
|
60
|
-
|
61
|
-
end
|
62
|
-
end
|
data/lib/pwrake/master.rb
DELETED
@@ -1,30 +0,0 @@
|
|
1
|
-
require "fileutils"
|
2
|
-
require "pathname"
|
3
|
-
require "yaml"
|
4
|
-
require "logger"
|
5
|
-
require "csv"
|
6
|
-
|
7
|
-
require "pwrake/version"
|
8
|
-
|
9
|
-
require "pwrake/iomux/runner"
|
10
|
-
require "pwrake/iomux/handler"
|
11
|
-
require "pwrake/iomux/handler_set"
|
12
|
-
require "pwrake/iomux/channel"
|
13
|
-
|
14
|
-
require "pwrake/logger"
|
15
|
-
require "pwrake/master/master"
|
16
|
-
require "pwrake/master/idle_cores"
|
17
|
-
require "pwrake/master/master_application"
|
18
|
-
require "pwrake/master/fiber_pool"
|
19
|
-
require "pwrake/option/option"
|
20
|
-
require "pwrake/option/option_filesystem"
|
21
|
-
require "pwrake/option/host_map"
|
22
|
-
require "pwrake/queue/queue_array"
|
23
|
-
require "pwrake/queue/task_queue"
|
24
|
-
require "pwrake/queue/locality_aware_queue"
|
25
|
-
require "pwrake/queue/no_action_queue"
|
26
|
-
require "pwrake/task/task_algorithm"
|
27
|
-
require "pwrake/task/task_manager"
|
28
|
-
require "pwrake/task/task_wrapper"
|
29
|
-
require "pwrake/task/task_property"
|
30
|
-
require "pwrake/task/task_rank"
|