pwrake 2.0.1 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +22 -9
  3. data/bin/gfwhere-pipe +33 -9
  4. data/bin/pwrake +5 -2
  5. data/bin/pwrake_branch +5 -3
  6. data/lib/pwrake/branch/branch.rb +95 -86
  7. data/lib/pwrake/branch/branch_application.rb +4 -0
  8. data/lib/pwrake/branch/communicator.rb +173 -0
  9. data/lib/pwrake/branch/communicator_set.rb +100 -0
  10. data/lib/pwrake/branch/fiber_queue.rb +10 -0
  11. data/lib/pwrake/branch/shell.rb +68 -24
  12. data/lib/pwrake/branch/shell_profiler.rb +2 -0
  13. data/lib/pwrake/gfarm/gfarm_postprocess.rb +8 -7
  14. data/lib/pwrake/logger.rb +5 -0
  15. data/lib/pwrake/master/master.rb +190 -87
  16. data/lib/pwrake/master/master_application.rb +8 -0
  17. data/lib/pwrake/nbio.rb +525 -0
  18. data/lib/pwrake/option/host_map.rb +36 -4
  19. data/lib/pwrake/option/option.rb +7 -1
  20. data/lib/pwrake/option/option_filesystem.rb +13 -3
  21. data/lib/pwrake/queue/locality_aware_queue.rb +41 -6
  22. data/lib/pwrake/queue/queue_array.rb +31 -11
  23. data/lib/pwrake/queue/task_queue.rb +15 -18
  24. data/lib/pwrake/report/report.rb +2 -0
  25. data/lib/pwrake/task/task_algorithm.rb +4 -1
  26. data/lib/pwrake/task/task_manager.rb +2 -0
  27. data/lib/pwrake/task/task_property.rb +1 -0
  28. data/lib/pwrake/task/task_wrapper.rb +40 -21
  29. data/lib/pwrake/version.rb +1 -1
  30. data/lib/pwrake/worker/invoker.rb +4 -29
  31. data/pwrake.gemspec +3 -2
  32. metadata +24 -12
  33. data/lib/pwrake/branch.rb +0 -22
  34. data/lib/pwrake/branch/worker_communicator.rb +0 -104
  35. data/lib/pwrake/iomux/channel.rb +0 -70
  36. data/lib/pwrake/iomux/handler.rb +0 -124
  37. data/lib/pwrake/iomux/handler_set.rb +0 -35
  38. data/lib/pwrake/iomux/runner.rb +0 -62
  39. data/lib/pwrake/master.rb +0 -30
@@ -1,35 +0,0 @@
1
- module Pwrake
2
-
3
- class HandlerSet < Array
4
-
5
- def kill(sig)
6
- each do |hdl|
7
- hdl.iow.puts("kill:#{sig}")
8
- hdl.iow.flush
9
- end
10
- end
11
-
12
- def exit
13
- a = []
14
- each do |hdl|
15
- iow = hdl.iow
16
- begin
17
- iow.puts "exit"
18
- iow.flush
19
- a << hdl
20
- Log.debug "HandlerSet#exit iow=#{iow.inspect}"
21
- rescue Errno::EPIPE => e
22
- if Rake.application.options.debug
23
- $stderr.puts "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}"
24
- $stderr.puts e.backtrace.join("\n")
25
- end
26
- Log.error "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}\n"+
27
- e.backtrace.join("\n")
28
- end
29
- end
30
- a.each{|hdl| hdl.wait_message("exited")}
31
- Log.debug "HandlerSet#exit end"
32
- end
33
-
34
- end
35
- end
@@ -1,62 +0,0 @@
1
- module Pwrake
2
-
3
- class TimeoutError < IOError; end
4
-
5
- class Runner
6
-
7
- def initialize(timeout=nil)
8
- @timeout = timeout
9
- @handler = {}
10
- @hb_time = {}
11
- end
12
-
13
- attr_reader :handler
14
-
15
- def add_handler(hdl)
16
- @handler[hdl.ior] ||= hdl
17
- end
18
-
19
- def delete_handler(hdl)
20
- @handler.delete(hdl.ior)
21
- end
22
-
23
- def run
24
- while !(io_set = @handler.keys).empty?
25
- sel, = IO.select(io_set,nil,nil,@timeout)
26
- if sel.nil?
27
- raise TimeoutError,"Timeout (#{@timeout} s) in IO.select"
28
- end
29
- sel.each do |io|
30
- @handler[io].process_line
31
- end
32
- if @timeout && @hb_earliest
33
- if Time.now - @hb_earliest > @timeout
34
- io = @hb_time.key(@hb_earliest)
35
- raise TimeoutError,"Timeout (#{@timeout}s) "+
36
- "in Heartbeat from host=#{get_host(io)}"
37
- end
38
- end
39
- end
40
- end
41
-
42
- def finish
43
- @handler.each do |io,hdl|
44
- hdl.finish
45
- end
46
- end
47
-
48
- # used to print an error message
49
- def get_host(io)
50
- hdl = @handler[io]
51
- h = hdl.respond_to?(:host) ? hdl.host : nil
52
- end
53
-
54
- # called when IO start and receive heartbeat
55
- def heartbeat(io)
56
- @hb_time[io] = Time.now
57
- @hb_earliest = @hb_time.values.min
58
- #Log.debug "heartbeat: host=#{get_host(io)}"
59
- end
60
-
61
- end
62
- end
data/lib/pwrake/master.rb DELETED
@@ -1,30 +0,0 @@
1
- require "fileutils"
2
- require "pathname"
3
- require "yaml"
4
- require "logger"
5
- require "csv"
6
-
7
- require "pwrake/version"
8
-
9
- require "pwrake/iomux/runner"
10
- require "pwrake/iomux/handler"
11
- require "pwrake/iomux/handler_set"
12
- require "pwrake/iomux/channel"
13
-
14
- require "pwrake/logger"
15
- require "pwrake/master/master"
16
- require "pwrake/master/idle_cores"
17
- require "pwrake/master/master_application"
18
- require "pwrake/master/fiber_pool"
19
- require "pwrake/option/option"
20
- require "pwrake/option/option_filesystem"
21
- require "pwrake/option/host_map"
22
- require "pwrake/queue/queue_array"
23
- require "pwrake/queue/task_queue"
24
- require "pwrake/queue/locality_aware_queue"
25
- require "pwrake/queue/no_action_queue"
26
- require "pwrake/task/task_algorithm"
27
- require "pwrake/task/task_manager"
28
- require "pwrake/task/task_wrapper"
29
- require "pwrake/task/task_property"
30
- require "pwrake/task/task_rank"