comana 0.0.9 → 0.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. data/CHANGES +20 -0
  2. data/Gemfile +10 -4
  3. data/Rakefile +26 -9
  4. data/VERSION +1 -1
  5. data/bin/machinestatus +189 -0
  6. data/bin/scpall +64 -0
  7. data/bin/sshall +58 -0
  8. data/example/dot.clustersetting +15 -0
  9. data/lib/comana.rb +12 -1
  10. data/lib/comana/clustersetting.rb +55 -0
  11. data/lib/comana/computationmanager.rb +10 -10
  12. data/lib/comana/hostinspector.rb +4 -0
  13. data/lib/comana/hostinspector/pbsnodes.rb +51 -0
  14. data/lib/comana/hostinspector/ping.rb +34 -0
  15. data/lib/comana/hostselector.rb +45 -0
  16. data/lib/comana/queuesubmitter.rb +39 -64
  17. data/memo.txt +2 -0
  18. data/test/helper.rb +17 -0
  19. data/{spec → test}/locked/input_a +0 -0
  20. data/{spec → test}/locked/input_b +0 -0
  21. data/{spec → test}/locked/lock_comana/dummy +0 -0
  22. data/{spec → test}/locked_outputted/input_a +0 -0
  23. data/{spec → test}/locked_outputted/input_b +0 -0
  24. data/{spec → test}/locked_outputted/lock_comana/dummy +0 -0
  25. data/{spec → test}/locked_outputted/output +0 -0
  26. data/{spec/queuesubmitter/locked/lock_queuesubmitter → test/not_executable}/dummy +0 -0
  27. data/{spec → test}/not_started/input_a +0 -0
  28. data/{spec → test}/not_started/input_b +0 -0
  29. data/{spec → test}/outputted/input_a +0 -0
  30. data/{spec → test}/outputted/input_b +0 -0
  31. data/{spec → test}/outputted/output +0 -0
  32. data/test/pbsnodes/Br09.xml +11 -0
  33. data/test/pbsnodes/Br10.xml +2 -0
  34. data/test/pbsnodes/Ge00.xml +2 -0
  35. data/test/pbsnodes/Ge08.xml +1 -0
  36. data/test/pbsnodes/all.xml +1 -0
  37. data/{spec/queuesubmitter/unlocked → test/queuesubmitter/locked/lock_queuesubmitter}/dummy +0 -0
  38. data/test/queuesubmitter/unlocked/dummy +0 -0
  39. data/test/test_clustersetting.rb +116 -0
  40. data/test/test_computationmanager.rb +131 -0
  41. data/test/test_hostinspector_pbsnodes.rb +92 -0
  42. data/test/test_hostinspector_ping.rb +21 -0
  43. data/test/test_hostselector.rb +57 -0
  44. data/test/test_queuesubmitter.rb +214 -0
  45. metadata +92 -49
  46. data/comana.gemspec +0 -84
  47. data/dot.machineinfo +0 -14
  48. data/lib/comana/machineinfo.rb +0 -44
  49. data/spec/computationmanager_spec.rb +0 -158
  50. data/spec/machineinfo +0 -7
  51. data/spec/machineinfo_spec.rb +0 -52
  52. data/spec/queuesubmitter_spec.rb +0 -263
  53. data/spec/spec_helper.rb +0 -12
data/CHANGES ADDED
@@ -0,0 +1,20 @@
1
+ = comana changelog
2
+
3
+ == Master (for 0.0.11)
4
+
5
+ == Version 0.0.10 [2014-08-28] released
6
+ - Provide namespace 'Comana' for classes.
7
+ - Add HostnameSelector
8
+ - Add HostnameSelector::Ping
9
+ - Add HostnameSelector::Pbsnodes
10
+ - MachineInfo class in machineinfo.rb is renamed to be
11
+ ClusterSetting class in clusterinfo.rb,
12
+ And change interface.
13
+ - Change Rspec to TestUnit
14
+ - Add bin/sshall
15
+ - Add bin/scpall
16
+ - Add priority option to opts in QueueSubmitter::initialize,
17
+ affecting PBS -p option in script.sh.
18
+ - ComputationManager.latest_modified_time is changed to public method.
19
+
20
+ == Version 0.0.9 [2012-05-29] released
data/Gemfile CHANGED
@@ -6,9 +6,15 @@ source "http://rubygems.org"
6
6
  # Add dependencies to develop your gem here.
7
7
  # Include everything needed to run rake, tests, features, etc.
8
8
  group :development do
9
- gem "rspec", ">= 2.10.0"
10
- gem "rdoc", ">= 3.12"
11
- gem "bundler", ">= 1.1.3"
12
- gem "jeweler", ">= 1.8.3"
9
+ #gem "rspec", ">= 2.10.0"
10
+ gem "rdoc", "~> 4.0.1"
11
+ gem "bundler", "~> 1.7.2"
12
+ gem "jeweler", ">= 2.0.1"
13
13
  gem "simplecov", ">= 0"
14
+ #gem "builder", "~> 3.2.2"
15
+ #gem "nokogiri", ">= 1.5.9"
16
+ #
17
+ gem "tefil", ">= 0.0.3"
18
+ #gem "psych", ">= 1.3.4"
19
+
14
20
  end
data/Rakefile CHANGED
@@ -31,18 +31,35 @@ Jeweler::Tasks.new do |gem|
31
31
  end
32
32
  Jeweler::RubygemsDotOrgTasks.new
33
33
 
34
- require 'rspec/core'
35
- require 'rspec/core/rake_task'
36
- RSpec::Core::RakeTask.new(:spec) do |spec|
37
- spec.pattern = FileList['spec/**/*_spec.rb']
38
- end
34
+ #require 'rspec/core'
35
+ #require 'rspec/core/rake_task'
36
+ #RSpec::Core::RakeTask.new(:spec) do |spec|
37
+ # spec.pattern = FileList['spec/**/*_spec.rb']
38
+ #end
39
+ #
40
+ #RSpec::Core::RakeTask.new(:rcov) do |spec|
41
+ # spec.pattern = 'spec/**/*_spec.rb'
42
+ # spec.rcov = true
43
+ #end
44
+ #
45
+ #task :default => :spec
39
46
 
40
- RSpec::Core::RakeTask.new(:rcov) do |spec|
41
- spec.pattern = 'spec/**/*_spec.rb'
42
- spec.rcov = true
47
+ require 'rake/testtask'
48
+ Rake::TestTask.new(:test) do |test|
49
+ test.libs << 'lib' << 'test'
50
+ test.pattern = 'test/**/test_*.rb'
51
+ test.verbose = true
43
52
  end
44
53
 
45
- task :default => :spec
54
+ #require 'rcov/rcovtask'
55
+ #Rcov::RcovTask.new do |test|
56
+ # test.libs << 'test'
57
+ # test.pattern = 'test/**/test_*.rb'
58
+ # test.verbose = true
59
+ # test.rcov_opts << '--exclude "gems/*"'
60
+ #end
61
+
62
+ task :default => :test
46
63
 
47
64
  require 'rdoc/task'
48
65
  Rake::RDocTask.new do |rdoc|
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.9
1
+ 0.0.10
data/bin/machinestatus ADDED
@@ -0,0 +1,189 @@
1
+ #! /usr/bin/env ruby
2
+ # coding: utf-8
3
+
4
+ USAGE = <<HERE
5
+ machinestatus HostA HostB
6
+ machinestatus -g GroupA GroupB #-g option interprets as group name
7
+ HERE
8
+
9
+ require "pp"
10
+ require "optparse"
11
+ require "rubygems"
12
+ require "comana"
13
+ require "highline"
14
+ require "tefil"
15
+
16
+ HIGHLINE = HighLine.new
17
+
18
+ ##Analyze options
19
+ OPTIONS = {}
20
+ op = OptionParser.new
21
+ op.on("-a", "--ping-alive", "Check only alive using ping"){OPTIONS[:ping] = true}
22
+ #op.on("-t", "--top" , "Info from top. "){OPTIONS[:top ] = true}
23
+ #op.on("-p", "--ps" , "Info from ps. "){OPTIONS[:ps ] = true}
24
+ op.on("-n", "--pbsnodes" , "Info from pbsnodes."){OPTIONS[:pbsnodes] = true}
25
+ op.on("-l", "--load" , "Pbsnodes; state, loadave."){
26
+ OPTIONS[:pbsnodes] = true
27
+ OPTIONS[:load] = true
28
+ }
29
+ #op.on("-d", "--dmesg" , "Info from dmesg. "){OPTIONS[:dmesg ] = true}
30
+
31
+ op.on("-g", "--group" , "Indicate group of hosts" ){OPTIONS[:group] = true}
32
+ op.on("-q", "--quiet" , "No output 'Wainting for...'"){OPTIONS[:quiet] = true}
33
+ op.parse!(ARGV)
34
+
35
+
36
+ ##Select hosts
37
+ cs = Comana::ClusterSetting.load_file
38
+ groups = {}
39
+ cs.groups.each do |key, val|
40
+ cluster = key
41
+ groups[key] = val["members"]
42
+ end
43
+
44
+ hs = Comana::HostSelector.new groups
45
+ if OPTIONS[:group]
46
+ hosts = []
47
+ ARGV.each do |group|
48
+ begin
49
+ hosts << hs.select_group(group)
50
+ rescue Comana::HostSelector::NoEntryError
51
+ $stderr.puts "Unknown group: #{group}"
52
+ end
53
+ end
54
+ hosts.flatten!
55
+ else
56
+ hosts = ARGV
57
+ end
58
+ hosts = hs.select_all if ARGV.empty?
59
+
60
+ #pp hosts;exit
61
+
62
+ ##Collect information
63
+ Thread.abort_on_exception = true
64
+ results = {}
65
+ threads = {}
66
+ hosts.each do |host|
67
+ threads[host] = Thread.start do
68
+ results[host] = {}
69
+ if OPTIONS[:ping]
70
+ results[host][:ping] = Comana::HostInspector::Ping.new host
71
+ end
72
+ if OPTIONS[:pbsnodes]
73
+ begin
74
+ results[host][:pbsnodes] = Comana::HostInspector::Pbsnodes.new(host, cs.pbs_server)
75
+ rescue Comana::HostInspector::Pbsnodes::UnknownNodeError
76
+ #results[host][:pbsnodes] = Comana::HostInspector::Pbsnodes.new(host, cs.pbs_server)
77
+ results[host][:pbsnodes] = nil
78
+ #results[host][:pbsnodes]["status"] = {}
79
+ end
80
+ end
81
+ end
82
+ end
83
+
84
+ threads.each do |host, thread|
85
+ print "Waiting for #{host}...\n" unless OPTIONS[:quiet]
86
+ thread.join # wait until all processes are completed
87
+ end
88
+
89
+ ##Titles
90
+ titles = []
91
+ titles << "host"
92
+ if OPTIONS[:ping]
93
+ titles << "ping"
94
+ end
95
+ if OPTIONS[:pbsnodes]
96
+ titles << "state"
97
+ titles << "np" unless OPTIONS[:load]
98
+ titles << "properties" unless OPTIONS[:load]
99
+ titles << "ntype" unless OPTIONS[:load]
100
+ titles << "gpus" unless OPTIONS[:load]
101
+ titles << "rectime" unless OPTIONS[:load]
102
+ titles << "varattr" unless OPTIONS[:load]
103
+ titles << "jobs"
104
+ titles << "state" unless OPTIONS[:load]
105
+ titles << "netload" unless OPTIONS[:load]
106
+ titles << "gres" unless OPTIONS[:load]
107
+ titles << "loadave"
108
+ titles << "ncpus" unless OPTIONS[:load]
109
+ titles << "physmem" unless OPTIONS[:load]
110
+ titles << "availmem" unless OPTIONS[:load]
111
+ titles << "totmem" unless OPTIONS[:load]
112
+ titles << "idletime" unless OPTIONS[:load]
113
+ titles << "nusers" unless OPTIONS[:load]
114
+ titles << "nsessions" unless OPTIONS[:load]
115
+ titles << "sessions" unless OPTIONS[:load]
116
+ titles << "uname" unless OPTIONS[:load]
117
+ titles << "opsys" unless OPTIONS[:load]
118
+ end
119
+
120
+ ##Data
121
+ data = []
122
+ results.keys.sort.each do |host|
123
+ items = [host]
124
+ #printf("%9s: ", )
125
+ #
126
+ if OPTIONS[:ping]
127
+ #state = HIGHLINE.color("dead", :red)
128
+ state = "dead"
129
+ state = "alive" if results[host][:ping].alive?
130
+ items << state
131
+ #sprintf("%5s", state)
132
+ end
133
+
134
+ if OPTIONS[:pbsnodes]
135
+ if results[host][:pbsnodes]
136
+ items << results[host][:pbsnodes].state
137
+ items << results[host][:pbsnodes].np unless OPTIONS[:load]
138
+ items << results[host][:pbsnodes].properties unless OPTIONS[:load]
139
+ items << results[host][:pbsnodes].ntype unless OPTIONS[:load]
140
+ items << results[host][:pbsnodes].gpus unless OPTIONS[:load]
141
+ items << results[host][:pbsnodes].status["rectime" ] unless OPTIONS[:load]
142
+ items << results[host][:pbsnodes].status["varattr" ] unless OPTIONS[:load]
143
+ items << results[host][:pbsnodes].status["jobs" ]
144
+ items << results[host][:pbsnodes].status["state" ] unless OPTIONS[:load]
145
+ items << results[host][:pbsnodes].status["netload" ] unless OPTIONS[:load]
146
+ items << results[host][:pbsnodes].status["gres" ] unless OPTIONS[:load]
147
+ items << results[host][:pbsnodes].status["loadave" ]
148
+ items << results[host][:pbsnodes].status["ncpus" ] unless OPTIONS[:load]
149
+ items << results[host][:pbsnodes].status["physmem" ] unless OPTIONS[:load]
150
+ items << results[host][:pbsnodes].status["availmem" ] unless OPTIONS[:load]
151
+ items << results[host][:pbsnodes].status["totmem" ] unless OPTIONS[:load]
152
+ items << results[host][:pbsnodes].status["idletime" ] unless OPTIONS[:load]
153
+ items << results[host][:pbsnodes].status["nusers" ] unless OPTIONS[:load]
154
+ items << results[host][:pbsnodes].status["nsessions"] unless OPTIONS[:load]
155
+ items << results[host][:pbsnodes].status["sessions" ] unless OPTIONS[:load]
156
+ items << results[host][:pbsnodes].status["uname" ] unless OPTIONS[:load]
157
+ items << results[host][:pbsnodes].status["opsys" ] unless OPTIONS[:load]
158
+ else
159
+ items << "pbs_unknown_host"
160
+ items << "" unless OPTIONS[:load]
161
+ items << "" unless OPTIONS[:load]
162
+ items << "" unless OPTIONS[:load]
163
+ items << "" unless OPTIONS[:load]
164
+ items << "" unless OPTIONS[:load]
165
+ items << "" unless OPTIONS[:load]
166
+ items << ""
167
+ items << "" unless OPTIONS[:load]
168
+ items << "" unless OPTIONS[:load]
169
+ items << "" unless OPTIONS[:load]
170
+ items << ""
171
+ items << "" unless OPTIONS[:load]
172
+ items << "" unless OPTIONS[:load]
173
+ items << "" unless OPTIONS[:load]
174
+ items << "" unless OPTIONS[:load]
175
+ items << "" unless OPTIONS[:load]
176
+ items << "" unless OPTIONS[:load]
177
+ items << "" unless OPTIONS[:load]
178
+ items << "" unless OPTIONS[:load]
179
+ items << "" unless OPTIONS[:load]
180
+ items << "" unless OPTIONS[:load]
181
+
182
+ end
183
+ end
184
+ data << items
185
+ end
186
+
187
+ data = data.unshift(titles)
188
+ #pp data
189
+ Tefil::ColumnFormer.form(data, $stdout, "|")
data/bin/scpall ADDED
@@ -0,0 +1,64 @@
1
+ #! /usr/bin/env ruby
2
+ # coding: utf-8
3
+
4
+ USAGE = <<HERE
5
+ scpall
6
+ scp from from on localhost to remotehost.
7
+
8
+ E.g.,
9
+ scpall -f from_file -t to_file HostA HostB
10
+ Indicate each host
11
+ scpall -f from_file -t to_file GroupA GroupB
12
+ Indicate Hosts with group names
13
+ scpall -f from_file -t to_file
14
+ Execute on all hosts when empty targets
15
+
16
+ scpall -u USER -f from_file -t to_file
17
+ Login as USER
18
+ HERE
19
+
20
+ require "optparse"
21
+ require "pp"
22
+ require "rubygems"
23
+ require "comana"
24
+
25
+ ## option analysis
26
+ OPTIONS = {}
27
+ op = OptionParser.new
28
+ op.on("-f from_file", "--from-file=str", "Copied file in a localhost."){|v| OPTIONS[:from_file] = v}
29
+ op.on("-t from_file", "--to-file=str", "File path in remotehosts."){|v| OPTIONS[:to_file] = v}
30
+ op.on("-g", "--group", "Interpret arguments as group name."){OPTIONS[:group] = true}
31
+ op.on("-u user", "--user=str" , "User on remote host."){|v| OPTIONS[:user] = v}
32
+ op.on("-T", "--test" , "Test mode; not execute."){OPTIONS[:test] = true}
33
+ op.parse!(ARGV)
34
+
35
+ unless OPTIONS[:from_file]
36
+ puts "-f option is not set."
37
+ puts USAGE; exit
38
+ end
39
+ unless OPTIONS[:to_file]
40
+ puts "-t option is not set."
41
+ puts USAGE; exit
42
+ end
43
+
44
+ hs = Comana::HostSelector.load_file
45
+ hosts = []
46
+ if ARGV.empty?
47
+ hosts = hs.select_all
48
+ elsif OPTIONS[:group]
49
+ ARGV.each do |tgt|
50
+ hosts = hs.select_group(tgt)
51
+ end
52
+ else
53
+ hosts = ARGV
54
+ end
55
+
56
+ hosts.each do |host|
57
+ host = "#{OPTIONS[:user]}@#{host}" if OPTIONS[:user]
58
+
59
+ command = "scp #{OPTIONS[:from_file]} #{host}:#{OPTIONS[:to_file]}"
60
+ print "#####{command}####"
61
+ puts
62
+ system command unless OPTIONS[:test]
63
+ end
64
+
data/bin/sshall ADDED
@@ -0,0 +1,58 @@
1
+ #! /usr/bin/env ruby
2
+ # coding: utf-8
3
+
4
+ USAGE = <<HERE
5
+ sshall
6
+ -g groups
7
+
8
+ E.g.,
9
+ sshall -c "ls -l /" HostA HostB
10
+ Indicate each host
11
+ sshall -c "ls -l /" -g GroupA GroupB
12
+ Indicate Hosts with group names
13
+ sshall -c "ls -l /"
14
+ Execute on all hosts when empty targets
15
+
16
+ sshall HostA HostB
17
+ Login each hosts when -c option is omitted.
18
+
19
+ sshall -u USER -c "ls -l /"
20
+ Login as USER
21
+ HERE
22
+
23
+ require "optparse"
24
+ require "pp"
25
+ require "rubygems"
26
+ require "comana"
27
+
28
+ ## option analysis
29
+ OPTIONS = {}
30
+ op = OptionParser.new
31
+ op.on("-c command" , "--command=str" , "Command to be sent."){|v| OPTIONS[:command] = v}
32
+ op.on("-g", "--group", "Interpret arguments as group name."){OPTIONS[:group] = true}
33
+ op.on("-u user", "--user=str" , "User on remote host."){|v| OPTIONS[:user] = v}
34
+ op.on("-T", "--test" , "Test mode; not execute."){OPTIONS[:test] = true}
35
+ op.parse!(ARGV)
36
+
37
+ hs = Comana::HostSelector.load_file
38
+
39
+ hosts = []
40
+ if ARGV.empty?
41
+ hosts = hs.select_all
42
+ elsif OPTIONS[:group]
43
+ ARGV.each do |tgt|
44
+ hosts = hs.select_group(tgt)
45
+ end
46
+ else
47
+ hosts = ARGV
48
+ end
49
+
50
+ hosts.each do |host|
51
+ host = "#{OPTIONS[:user]}@#{host}" if OPTIONS[:user]
52
+
53
+ command = "ssh #{host} #{OPTIONS[:command]}"
54
+ print "#####{command}####"
55
+ puts
56
+ system command unless OPTIONS[:test]
57
+ end
58
+
@@ -0,0 +1,15 @@
1
+ --- # vim: syntax=yaml
2
+ # cluster_name: [ num_economy, num_speed ]
3
+
4
+ pbs_server: "P00"
5
+
6
+ groups:
7
+ A:
8
+ data1: 'A-1'
9
+ data2: 'A-2'
10
+ members: ["A00", "A01"]
11
+
12
+ B:
13
+ data1: 'B-1'
14
+ data2: 'B-2'
15
+ members: ["B00", "B01", "B02"]
data/lib/comana.rb CHANGED
@@ -1,2 +1,13 @@
1
+ module Comana; end
2
+
3
+ #require "rubygems"
4
+ #gem "nokogiri"
5
+ #require "nokogiri"
6
+
1
7
  require "comana/computationmanager.rb"
2
- require "comana/machineinfo.rb"
8
+ require "comana/clustersetting.rb"
9
+ require "comana/hostselector.rb"
10
+ require "comana/queuesubmitter.rb"
11
+ require "comana/hostinspector.rb"
12
+ require "comana/hostinspector/ping.rb"
13
+ require "comana/hostinspector/pbsnodes.rb"
@@ -0,0 +1,55 @@
1
+ #! /usr/bin/env ruby
2
+ # coding: utf-8
3
+
4
+ require "yaml"
5
+
6
+ # Series name is composed only of alphabets.
7
+ # Host name is started by the series name and followed by integers.
8
+ # E.g.,
9
+ # "Fe", "Fe00", "Fe01" are of series "Fe" and not "F"
10
+ class Comana::ClusterSetting
11
+ attr_reader :groups, :pbs_server
12
+
13
+ DEFAULT_DATA_FILE = ENV["HOME"] + "/.clustersetting"
14
+
15
+ class NoEntryError < Exception; end
16
+
17
+ #
18
+ def initialize(settings)
19
+ @pbs_server = settings["pbs_server"]
20
+ @groups = settings["groups"]
21
+ end
22
+
23
+ def self.load_file(data_file = DEFAULT_DATA_FILE)
24
+ settings = YAML.load_file(data_file)
25
+ self.new settings
26
+ end
27
+
28
+ #Return belonged cluster of the host.
29
+ #Raise NoEntryError if not match.
30
+ def belonged_cluster(hostname)
31
+ @groups.each do |group, settings|
32
+ next unless settings["members"]
33
+ return group if settings["members"].include? hostname
34
+ end
35
+ raise NoEntryError, "#{hostname} is not in `@groups': #{@groups.inspect}"
36
+ end
37
+
38
+ #Return settings as a hash for a cluster.
39
+ def settings_group(clustername)
40
+ @groups[clustername]
41
+ end
42
+
43
+ #Return settings as a hash for a host belonged to cluster.
44
+ def settings_host(hostname)
45
+ settings_group(belonged_cluster(hostname))
46
+ end
47
+
48
+ #Return an array of cluster names.
49
+ def clusters
50
+ @groups.keys
51
+ end
52
+
53
+
54
+ end
55
+