pwrake 2.0.1 → 2.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +22 -9
  3. data/bin/gfwhere-pipe +33 -9
  4. data/bin/pwrake +5 -2
  5. data/bin/pwrake_branch +5 -3
  6. data/lib/pwrake/branch/branch.rb +95 -86
  7. data/lib/pwrake/branch/branch_application.rb +4 -0
  8. data/lib/pwrake/branch/communicator.rb +173 -0
  9. data/lib/pwrake/branch/communicator_set.rb +100 -0
  10. data/lib/pwrake/branch/fiber_queue.rb +10 -0
  11. data/lib/pwrake/branch/shell.rb +68 -24
  12. data/lib/pwrake/branch/shell_profiler.rb +2 -0
  13. data/lib/pwrake/gfarm/gfarm_postprocess.rb +8 -7
  14. data/lib/pwrake/logger.rb +5 -0
  15. data/lib/pwrake/master/master.rb +190 -87
  16. data/lib/pwrake/master/master_application.rb +8 -0
  17. data/lib/pwrake/nbio.rb +525 -0
  18. data/lib/pwrake/option/host_map.rb +36 -4
  19. data/lib/pwrake/option/option.rb +7 -1
  20. data/lib/pwrake/option/option_filesystem.rb +13 -3
  21. data/lib/pwrake/queue/locality_aware_queue.rb +41 -6
  22. data/lib/pwrake/queue/queue_array.rb +31 -11
  23. data/lib/pwrake/queue/task_queue.rb +15 -18
  24. data/lib/pwrake/report/report.rb +2 -0
  25. data/lib/pwrake/task/task_algorithm.rb +4 -1
  26. data/lib/pwrake/task/task_manager.rb +2 -0
  27. data/lib/pwrake/task/task_property.rb +1 -0
  28. data/lib/pwrake/task/task_wrapper.rb +40 -21
  29. data/lib/pwrake/version.rb +1 -1
  30. data/lib/pwrake/worker/invoker.rb +4 -29
  31. data/pwrake.gemspec +3 -2
  32. metadata +24 -12
  33. data/lib/pwrake/branch.rb +0 -22
  34. data/lib/pwrake/branch/worker_communicator.rb +0 -104
  35. data/lib/pwrake/iomux/channel.rb +0 -70
  36. data/lib/pwrake/iomux/handler.rb +0 -124
  37. data/lib/pwrake/iomux/handler_set.rb +0 -35
  38. data/lib/pwrake/iomux/runner.rb +0 -62
  39. data/lib/pwrake/master.rb +0 -30
@@ -1,35 +0,0 @@
1
- module Pwrake
2
-
3
- class HandlerSet < Array
4
-
5
- def kill(sig)
6
- each do |hdl|
7
- hdl.iow.puts("kill:#{sig}")
8
- hdl.iow.flush
9
- end
10
- end
11
-
12
- def exit
13
- a = []
14
- each do |hdl|
15
- iow = hdl.iow
16
- begin
17
- iow.puts "exit"
18
- iow.flush
19
- a << hdl
20
- Log.debug "HandlerSet#exit iow=#{iow.inspect}"
21
- rescue Errno::EPIPE => e
22
- if Rake.application.options.debug
23
- $stderr.puts "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}"
24
- $stderr.puts e.backtrace.join("\n")
25
- end
26
- Log.error "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}\n"+
27
- e.backtrace.join("\n")
28
- end
29
- end
30
- a.each{|hdl| hdl.wait_message("exited")}
31
- Log.debug "HandlerSet#exit end"
32
- end
33
-
34
- end
35
- end
@@ -1,62 +0,0 @@
1
- module Pwrake
2
-
3
- class TimeoutError < IOError; end
4
-
5
- class Runner
6
-
7
- def initialize(timeout=nil)
8
- @timeout = timeout
9
- @handler = {}
10
- @hb_time = {}
11
- end
12
-
13
- attr_reader :handler
14
-
15
- def add_handler(hdl)
16
- @handler[hdl.ior] ||= hdl
17
- end
18
-
19
- def delete_handler(hdl)
20
- @handler.delete(hdl.ior)
21
- end
22
-
23
- def run
24
- while !(io_set = @handler.keys).empty?
25
- sel, = IO.select(io_set,nil,nil,@timeout)
26
- if sel.nil?
27
- raise TimeoutError,"Timeout (#{@timeout} s) in IO.select"
28
- end
29
- sel.each do |io|
30
- @handler[io].process_line
31
- end
32
- if @timeout && @hb_earliest
33
- if Time.now - @hb_earliest > @timeout
34
- io = @hb_time.key(@hb_earliest)
35
- raise TimeoutError,"Timeout (#{@timeout}s) "+
36
- "in Heartbeat from host=#{get_host(io)}"
37
- end
38
- end
39
- end
40
- end
41
-
42
- def finish
43
- @handler.each do |io,hdl|
44
- hdl.finish
45
- end
46
- end
47
-
48
- # used to print an error message
49
- def get_host(io)
50
- hdl = @handler[io]
51
- h = hdl.respond_to?(:host) ? hdl.host : nil
52
- end
53
-
54
- # called when IO start and receive heartbeat
55
- def heartbeat(io)
56
- @hb_time[io] = Time.now
57
- @hb_earliest = @hb_time.values.min
58
- #Log.debug "heartbeat: host=#{get_host(io)}"
59
- end
60
-
61
- end
62
- end
data/lib/pwrake/master.rb DELETED
@@ -1,30 +0,0 @@
1
- require "fileutils"
2
- require "pathname"
3
- require "yaml"
4
- require "logger"
5
- require "csv"
6
-
7
- require "pwrake/version"
8
-
9
- require "pwrake/iomux/runner"
10
- require "pwrake/iomux/handler"
11
- require "pwrake/iomux/handler_set"
12
- require "pwrake/iomux/channel"
13
-
14
- require "pwrake/logger"
15
- require "pwrake/master/master"
16
- require "pwrake/master/idle_cores"
17
- require "pwrake/master/master_application"
18
- require "pwrake/master/fiber_pool"
19
- require "pwrake/option/option"
20
- require "pwrake/option/option_filesystem"
21
- require "pwrake/option/host_map"
22
- require "pwrake/queue/queue_array"
23
- require "pwrake/queue/task_queue"
24
- require "pwrake/queue/locality_aware_queue"
25
- require "pwrake/queue/no_action_queue"
26
- require "pwrake/task/task_algorithm"
27
- require "pwrake/task/task_manager"
28
- require "pwrake/task/task_wrapper"
29
- require "pwrake/task/task_property"
30
- require "pwrake/task/task_rank"