comana 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES +52 -19
- data/Gemfile +1 -0
- data/README.rdoc +6 -1
- data/VERSION +1 -1
- data/bin/genqsub +38 -30
- data/bin/scpall +1 -1
- data/comana.gemspec +19 -15
- data/example/dot.clustersetting +24 -9
- data/lib/comana/clustersetting.rb +10 -0
- data/lib/comana/computationmanager.rb +134 -10
- data/lib/comana/gridengine.rb +173 -0
- data/lib/comana/hostinspector.rb +5 -59
- data/lib/comana.rb +4 -7
- data/test/gridengine/qconfsql.dat +13 -0
- data/test/gridengine/qstatf0.xml +1275 -0
- data/test/gridengine/qstatf1.xml +39 -0
- data/test/gridengine/qstatq0.xml +247 -0
- data/test/gridengine/qstatq1.xml +37 -0
- data/test/gridengine/qstatu0.xml +437 -0
- data/test/gridengine/qstatu1.xml +47 -0
- data/test/hostselector/dot.clustersetting +15 -0
- data/test/test_clustersetting.rb +20 -23
- data/test/test_computationmanager.rb +49 -15
- data/test/test_gridengine.rb +198 -0
- data/test/test_hostinspector.rb +11 -29
- data/test/test_hostselector.rb +2 -13
- metadata +27 -14
- data/bin/machinestatus +0 -192
- data/bin/queueinfo +0 -28
- data/lib/comana/gridenginescript.rb +0 -68
- data/lib/comana/queuemanager.rb +0 -34
- data/lib/comana/queuesubmitter.rb +0 -156
- data/memo.txt +0 -34
- data/test/test_gridenginescript.rb +0 -16
- data/test/test_queuemanager.rb +0 -19
- data/test/test_queuesubmitter.rb +0 -214
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 26543f302d5fb0a59138a6542e84caef0b2491ce
|
4
|
+
data.tar.gz: 2f0061d8200ae6ac68a18c0616a82100c96e8cae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3a797bd56b438b3142b1836f76ff94d432164b9d5d5d3e3929490460299e5a102b9365a4c9a79b6eea3ed29b1f17ede8ebd125da99b81d404faf661c7c954253
|
7
|
+
data.tar.gz: c4029c86b02d2ea9774351f8988b7cd48896355ff4c7fa08c9a0268b0308cdf814ef4c7ea8a65c7a3e31962883f93148c66f69111b3e6073b1fed50be0eff124
|
data/CHANGES
CHANGED
@@ -1,27 +1,60 @@
|
|
1
1
|
= comana changelog
|
2
2
|
|
3
|
-
== Master (for 0.1.
|
3
|
+
== Master (for 0.1.2)
|
4
|
+
|
5
|
+
== Version 0.1.1 [2016-03-11] released
|
6
|
+
- Comana::ComputationManager#start() rename to ...
|
7
|
+
- Change argument specification of bin/genqsub
|
8
|
+
- Delete GridEngineScript
|
9
|
+
- Add GridEngine
|
10
|
+
- Add ComputationManager::qsub
|
11
|
+
- Delete bin/machinestatus. Use hostinfo
|
12
|
+
- Delete QueueSubmitter
|
4
13
|
|
5
14
|
== Version 0.1.0 [2016-03-04] released
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
15
|
+
- Comana::ComputationManager#start() rename to exec().
|
16
|
+
- Comana::ComputationManager.exec is added.
|
17
|
+
- Add Comana::HostInspector
|
18
|
+
- Add bin/hostinfo
|
19
|
+
- Add bin/genqsub
|
11
20
|
|
12
21
|
== Version 0.0.10 [2014-08-28] released
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
22
|
+
- Provide namespace 'Comana' for classes.
|
23
|
+
- Add HostnameSelector
|
24
|
+
- Add HostnameSelector::Ping
|
25
|
+
- Add HostnameSelector::Pbsnodes
|
26
|
+
- MachineInfo class in machineinfo.rb is renamed to be
|
27
|
+
ClusterSetting class in clusterinfo.rb,
|
28
|
+
And change interface.
|
29
|
+
- Change Rspec to TestUnit
|
30
|
+
- Add bin/sshall
|
31
|
+
- Add bin/scpall
|
32
|
+
- Add priority option to opts in QueueSubmitter::initialize,
|
33
|
+
affecting PBS -p option in script.sh.
|
34
|
+
- ComputationManager.latest_modified_time is changed to public method.
|
26
35
|
|
27
36
|
== Version 0.0.9 [2012-05-29] released
|
37
|
+
Change opts specification in QueueSubmitter; symbol to string
|
38
|
+
|
39
|
+
== Version 0.0.8
|
40
|
+
Changed queuesubmitter option. (long name symbol)
|
41
|
+
Add prologue and epilogue to QueueSubmitter.
|
42
|
+
|
43
|
+
== Version 0.0.7
|
44
|
+
Bugfix of danger to rsync large directory:
|
45
|
+
- bin/submitqueue is removed.
|
46
|
+
- lib/queuesubmitter.rb is modified.
|
47
|
+
lib/queuesubmitter.rb is modified.
|
48
|
+
- finished? returns bool.
|
49
|
+
- content of qsub script to use "rsync" and "mv dir ~/.trash".
|
50
|
+
|
51
|
+
== Version 0.0.6
|
52
|
+
bin/submitqueue is add.
|
53
|
+
lib/queuesubmitter.rb is add.
|
54
|
+
|
55
|
+
== Version 0.0.5
|
56
|
+
Directory structure and file names are changed.
|
57
|
+
Method names are hanged.
|
58
|
+
MachineInfo is added.
|
59
|
+
|
60
|
+
== Version 0.0.4
|
data/Gemfile
CHANGED
data/README.rdoc
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
= comana
|
2
2
|
|
3
|
-
|
3
|
+
Comana: COmputation MANAger
|
4
|
+
|
5
|
+
Comana don't have information of output files.
|
6
|
+
Because it is difficult to define final output and to deal systematically.
|
7
|
+
e.g., repeated calculation till convergence.
|
8
|
+
|
4
9
|
|
5
10
|
== Contributing to comana
|
6
11
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.1
|
data/bin/genqsub
CHANGED
@@ -5,41 +5,49 @@ require "pp"
|
|
5
5
|
require "comana"
|
6
6
|
require "optparse"
|
7
7
|
|
8
|
-
|
9
|
-
OPTIONS = {}
|
8
|
+
options = {}
|
10
9
|
op = OptionParser.new
|
11
|
-
op.banner =
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
op.on(
|
10
|
+
op.banner = <<HERE
|
11
|
+
genqsub [options] [dirs]
|
12
|
+
Submit queue to grid engine. Empty 'dirs' in argument indicates current directory.
|
13
|
+
HERE
|
14
|
+
op.on('--q-name=queue' , "Queue name for '#$ -q'"){
|
15
|
+
|v| options[:q_name] = v}
|
16
|
+
op.on('--pe-name=pe' , "Parallel environment name for '\#$ -pe'"){
|
17
|
+
|v| options[:pe_name] = v}
|
18
|
+
op.on('--ppn=ppn' , "Process per node, 2nd argument for'#$ -pe'"){
|
19
|
+
|v| options[:ppn] = v}
|
20
|
+
op.on('--ld_library_path=path', "Set 'LD_LIBRARY_PATH'"){
|
21
|
+
|v| options[:ld_library_path] = v}
|
22
|
+
op.on('--load-group=group' , "Load setting from group in setting file."){
|
23
|
+
|v| options[:load_group] = v}
|
24
|
+
#op.on('-no_submit' , "Write script', ' but not submit."){
|
25
|
+
#|v| options[:no_submit] = v}
|
26
|
+
op.on('--auto' , "Select low load group and automatic setting."){ options[:auto] = true}
|
16
27
|
op.parse!(ARGV)
|
17
28
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
29
|
+
if options[:load_group]
|
30
|
+
settings = Comana::ClusterSetting.load_file
|
31
|
+
gs = settings.groups[options[:load_group]]
|
32
|
+
q_name ||= gs['queue']
|
33
|
+
pe_name ||= gs['pe']
|
34
|
+
ppn ||= gs['ppn']
|
35
|
+
ld_library_path ||= gs['ld_library_path']
|
36
|
+
end
|
25
37
|
|
26
|
-
|
27
|
-
puts "
|
28
|
-
puts op.banner
|
38
|
+
if ARGV.empty?
|
39
|
+
puts "Need command in argument. Exit."
|
29
40
|
exit
|
41
|
+
else
|
42
|
+
command = ARGV.join(" ")
|
30
43
|
end
|
31
|
-
ppn = settings["ppn"]
|
32
44
|
|
33
|
-
|
45
|
+
GridEngine.write_qsub_script(
|
46
|
+
q_name: q_name ,
|
47
|
+
pe_name: pe_name ,
|
48
|
+
ppn: ppn ,
|
49
|
+
command: command ,
|
50
|
+
ld_library_path: ld_library_path ,
|
51
|
+
io: $stdout
|
52
|
+
)
|
34
53
|
|
35
|
-
GridEngineScript.generate(io, series, ppn, command)
|
36
|
-
io.close
|
37
|
-
|
38
|
-
if OPTIONS[:submit]
|
39
|
-
unless OPTIONS[:outfile]
|
40
|
-
puts "'--submit' option must use with '--outfile' option."
|
41
|
-
exit
|
42
|
-
end
|
43
|
-
|
44
|
-
GridEngineScript.write_submit(series, ppn, command, OPTIONS[:outfile])
|
45
|
-
end
|
data/bin/scpall
CHANGED
@@ -26,7 +26,7 @@ require "comana"
|
|
26
26
|
OPTIONS = {}
|
27
27
|
op = OptionParser.new
|
28
28
|
op.on("-f from_file", "--from-file=str", "Copied file in a localhost."){|v| OPTIONS[:from_file] = v}
|
29
|
-
op.on("-t
|
29
|
+
op.on("-t to_file", "--to-file=str", "File path in remotehosts."){|v| OPTIONS[:to_file] = v}
|
30
30
|
op.on("-g", "--group", "Interpret arguments as group name."){OPTIONS[:group] = true}
|
31
31
|
op.on("-u user", "--user=str" , "User on remote host."){|v| OPTIONS[:user] = v}
|
32
32
|
op.on("-T", "--test" , "Test mode; not execute."){OPTIONS[:test] = true}
|
data/comana.gemspec
CHANGED
@@ -2,19 +2,19 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: comana 0.1.
|
5
|
+
# stub: comana 0.1.1 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "comana"
|
9
|
-
s.version = "0.1.
|
9
|
+
s.version = "0.1.1"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["ippei94da"]
|
14
|
-
s.date = "2016-03-
|
14
|
+
s.date = "2016-03-10"
|
15
15
|
s.description = "Comana, COmputation MANAger,\n is a software to provide a framework of\n managing scientific computing.\n Researchers on computing have to check calculation and\n generate new calculation and execute, repeatedly.\n The abstract class that this gem provide would help the work.\n "
|
16
16
|
s.email = "ippei94da@gmail.com"
|
17
|
-
s.executables = ["genqsub", "hostinfo", "
|
17
|
+
s.executables = ["genqsub", "hostinfo", "scpall", "sshall"]
|
18
18
|
s.extra_rdoc_files = [
|
19
19
|
"LICENSE.txt",
|
20
20
|
"README.rdoc"
|
@@ -30,8 +30,6 @@ Gem::Specification.new do |s|
|
|
30
30
|
"VERSION",
|
31
31
|
"bin/genqsub",
|
32
32
|
"bin/hostinfo",
|
33
|
-
"bin/machinestatus",
|
34
|
-
"bin/queueinfo",
|
35
33
|
"bin/scpall",
|
36
34
|
"bin/sshall",
|
37
35
|
"comana.gemspec",
|
@@ -39,14 +37,19 @@ Gem::Specification.new do |s|
|
|
39
37
|
"lib/comana.rb",
|
40
38
|
"lib/comana/clustersetting.rb",
|
41
39
|
"lib/comana/computationmanager.rb",
|
42
|
-
"lib/comana/
|
40
|
+
"lib/comana/gridengine.rb",
|
43
41
|
"lib/comana/hostinspector.rb",
|
44
42
|
"lib/comana/hostselector.rb",
|
45
|
-
"
|
46
|
-
"
|
47
|
-
"
|
43
|
+
"test/gridengine/qconfsql.dat",
|
44
|
+
"test/gridengine/qstatf0.xml",
|
45
|
+
"test/gridengine/qstatf1.xml",
|
46
|
+
"test/gridengine/qstatq0.xml",
|
47
|
+
"test/gridengine/qstatq1.xml",
|
48
|
+
"test/gridengine/qstatu0.xml",
|
49
|
+
"test/gridengine/qstatu1.xml",
|
48
50
|
"test/helper.rb",
|
49
51
|
"test/hostinspector/.gitignore",
|
52
|
+
"test/hostselector/dot.clustersetting",
|
50
53
|
"test/locked/input_a",
|
51
54
|
"test/locked/input_b",
|
52
55
|
"test/locked/lock_comana/dummy",
|
@@ -69,15 +72,13 @@ Gem::Specification.new do |s|
|
|
69
72
|
"test/queuesubmitter/unlocked/dummy",
|
70
73
|
"test/test_clustersetting.rb",
|
71
74
|
"test/test_computationmanager.rb",
|
72
|
-
"test/
|
75
|
+
"test/test_gridengine.rb",
|
73
76
|
"test/test_hostinspector.rb",
|
74
|
-
"test/test_hostselector.rb"
|
75
|
-
"test/test_queuemanager.rb",
|
76
|
-
"test/test_queuesubmitter.rb"
|
77
|
+
"test/test_hostselector.rb"
|
77
78
|
]
|
78
79
|
s.homepage = "http://github.com/ippei94da/comana"
|
79
80
|
s.licenses = ["MIT"]
|
80
|
-
s.rubygems_version = "2.
|
81
|
+
s.rubygems_version = "2.5.1"
|
81
82
|
s.summary = "Manager for scientific computing"
|
82
83
|
|
83
84
|
if s.respond_to? :specification_version then
|
@@ -90,6 +91,7 @@ Gem::Specification.new do |s|
|
|
90
91
|
s.add_development_dependency(%q<jeweler>, ["~> 2.0"])
|
91
92
|
s.add_development_dependency(%q<simplecov>, ["~> 0.11"])
|
92
93
|
s.add_development_dependency(%q<tefil>, [">= 0.1"])
|
94
|
+
s.add_development_dependency(%q<builtinextension>, [">= 0.1"])
|
93
95
|
else
|
94
96
|
s.add_dependency(%q<test-unit>, ["~> 3.1"])
|
95
97
|
s.add_dependency(%q<rdoc>, ["~> 4.2"])
|
@@ -97,6 +99,7 @@ Gem::Specification.new do |s|
|
|
97
99
|
s.add_dependency(%q<jeweler>, ["~> 2.0"])
|
98
100
|
s.add_dependency(%q<simplecov>, ["~> 0.11"])
|
99
101
|
s.add_dependency(%q<tefil>, [">= 0.1"])
|
102
|
+
s.add_dependency(%q<builtinextension>, [">= 0.1"])
|
100
103
|
end
|
101
104
|
else
|
102
105
|
s.add_dependency(%q<test-unit>, ["~> 3.1"])
|
@@ -105,6 +108,7 @@ Gem::Specification.new do |s|
|
|
105
108
|
s.add_dependency(%q<jeweler>, ["~> 2.0"])
|
106
109
|
s.add_dependency(%q<simplecov>, ["~> 0.11"])
|
107
110
|
s.add_dependency(%q<tefil>, [">= 0.1"])
|
111
|
+
s.add_dependency(%q<builtinextension>, [">= 0.1"])
|
108
112
|
end
|
109
113
|
end
|
110
114
|
|
data/example/dot.clustersetting
CHANGED
@@ -4,12 +4,27 @@
|
|
4
4
|
pbs_server: "P00"
|
5
5
|
|
6
6
|
groups:
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
7
|
+
Ag:
|
8
|
+
mpi: '/opt/openmpi-intel/bin/mpiexec'
|
9
|
+
vasp: '/opt/bin/vasp5212openmpi'
|
10
|
+
ppn: 4
|
11
|
+
members: [
|
12
|
+
'Ag00','Ag01','Ag02','Ag03','Ag04','Ag05','Ag06','Ag07'
|
13
|
+
]
|
14
|
+
ld_library_path: '/usr/lib:/usr/local/lib:/opt/intel/mkl/lib/intel64:/opt/intel/lib/intel64:/opt/intel/lib:/opt/openmpi-intel/lib'
|
15
|
+
queue: 'Ag.q'
|
16
|
+
pe: 'Ag.openmpi'
|
17
|
+
benchmark: 11.6302
|
18
|
+
|
19
|
+
Cd:
|
20
|
+
mpi: '/opt/openmpi-intel/bin/mpiexec'
|
21
|
+
vasp: '/opt/bin/vasp5212openmpi'
|
22
|
+
ppn: 4
|
23
|
+
members: [
|
24
|
+
'Cd00','Cd01','Cd02','Cd03','Cd04','Cd05','Cd06','Cd07'
|
25
|
+
]
|
26
|
+
ld_library_path: '/usr/lib:/usr/local/lib:/opt/intel/mkl/lib/intel64:/opt/intel/lib/intel64:/opt/intel/lib:/opt/openmpi-intel/lib'
|
27
|
+
queue: 'Cd.q'
|
28
|
+
pe: 'Cd.openmpi'
|
29
|
+
benchmark: 12.0019
|
30
|
+
|
@@ -41,6 +41,16 @@ class Comana::ClusterSetting
|
|
41
41
|
@groups[clustername]
|
42
42
|
end
|
43
43
|
|
44
|
+
#Return settings as a hash for a cluster, the 'queue' key has a value of q_name.
|
45
|
+
def settings_queue(q_name)
|
46
|
+
result = nil
|
47
|
+
#pp @groups
|
48
|
+
@groups.each do |name, items|
|
49
|
+
result = items if items['queue'] == q_name
|
50
|
+
end
|
51
|
+
result
|
52
|
+
end
|
53
|
+
|
44
54
|
#Return settings as a hash for a host belonged to cluster.
|
45
55
|
def settings_host(hostname)
|
46
56
|
settings_group(belonged_cluster(hostname))
|
@@ -5,9 +5,14 @@
|
|
5
5
|
# Users have to redefine some methods in subclasses for various computation.
|
6
6
|
#
|
7
7
|
class Comana::ComputationManager
|
8
|
-
class
|
9
|
-
class
|
10
|
-
class
|
8
|
+
class InitializeError < StandardError; end
|
9
|
+
class NotImplementedError < StandardError; end
|
10
|
+
class AlreadyStartedError < StandardError; end
|
11
|
+
class AlreadySubmittedError < StandardError; end
|
12
|
+
class ExecuteError < StandardError; end
|
13
|
+
|
14
|
+
QSUB_SCRIPT_NAME = 'qsub.sh'
|
15
|
+
QSUB_LOG_NAME = 'qsub.log'
|
11
16
|
|
12
17
|
attr_reader :dir
|
13
18
|
|
@@ -18,28 +23,121 @@ class Comana::ComputationManager
|
|
18
23
|
@alive_time = 3600
|
19
24
|
end
|
20
25
|
|
21
|
-
def self.
|
26
|
+
def self.execute(args)
|
22
27
|
targets = args
|
23
|
-
targets = [ENV['PWD']] if targets.
|
28
|
+
targets = [ENV['PWD']] if targets.empty?
|
24
29
|
|
25
30
|
targets.each do |dir|
|
26
31
|
print "#{dir}..."
|
27
32
|
begin
|
28
33
|
calc_dir = self.new(dir)
|
29
34
|
rescue => exc
|
30
|
-
puts "Not
|
35
|
+
puts "Not #{self}: #{dir}"
|
31
36
|
next
|
32
37
|
end
|
33
38
|
|
34
39
|
begin
|
35
40
|
calc_dir.start
|
36
|
-
rescue
|
37
|
-
puts "Already started
|
41
|
+
rescue self::AlreadyStartedError
|
42
|
+
puts "Already started: #{dir}"
|
38
43
|
next
|
39
44
|
end
|
40
45
|
end
|
41
46
|
end
|
42
47
|
|
48
|
+
def self.qsub(args, options)
|
49
|
+
if options[:auto] || options[:load_group]
|
50
|
+
# OK
|
51
|
+
elsif !(options[:q_name] && options[:pe_name] && options[:ppn] && options[:command])
|
52
|
+
puts "Lack of required options."
|
53
|
+
puts "Need (--auto) or (--load-group) or (--q-name && --pe-name && --ppn && --command )"
|
54
|
+
puts "E.g., OK: #{File.basename($0)} --auto"
|
55
|
+
puts " OK: #{File.basename($0)} --load-group=cluster_name"
|
56
|
+
puts " OK: #{File.basename($0)} --q-name=a --pe-name=b --ppn=1 --command=c"
|
57
|
+
puts " NG: #{File.basename($0)} --q-name=a --pe-name=b --ppn=1"
|
58
|
+
puts "Exit."
|
59
|
+
exit
|
60
|
+
end
|
61
|
+
|
62
|
+
tgts = args
|
63
|
+
tgts = [ENV['PWD']] if tgts.empty?
|
64
|
+
|
65
|
+
tgts.each do |dir|
|
66
|
+
cs = Comana::ClusterSetting.load_file
|
67
|
+
if options[:load_group]
|
68
|
+
q_name = options[:load_group]
|
69
|
+
elsif options[:auto]
|
70
|
+
queues = Comana::GridEngine.queues
|
71
|
+
jobs = {}
|
72
|
+
hosts = {}
|
73
|
+
benchmarks = {}
|
74
|
+
queues.each do |q|
|
75
|
+
jobs[q] = Comana::GridEngine.queue_jobs(q).size
|
76
|
+
hosts[q] = Comana::GridEngine.queue_alive_nums[q]
|
77
|
+
benchmarks[q] = Comana::ClusterSetting.load_file.settings_queue(q)['benchmark']
|
78
|
+
end
|
79
|
+
q_name = self.effective_queue(queues, jobs, hosts, benchmarks)
|
80
|
+
end
|
81
|
+
|
82
|
+
if options[:load_group] || options[:auto]
|
83
|
+
gs = cs.settings_queue(q_name)
|
84
|
+
q_name ||= gs['queue']
|
85
|
+
pe_name ||= gs['pe']
|
86
|
+
ppn ||= gs['ppn']
|
87
|
+
ld_library_path ||= gs['ld_library_path']
|
88
|
+
end
|
89
|
+
|
90
|
+
q_name = options[:q_name] if options[:q_name]
|
91
|
+
pe_name = options[:pe_name] if options[:pe_name]
|
92
|
+
ppn = options[:ppn] if options[:ppn]
|
93
|
+
ld_library_path = options[:ld_library_path] if options[:ld_library_path]
|
94
|
+
command = options[:command] || "#{`which #{__FILE__}`.chomp} execute"
|
95
|
+
|
96
|
+
begin
|
97
|
+
calc_dir = self.new(dir)
|
98
|
+
calc_dir.queue_submit(
|
99
|
+
q_name: q_name,
|
100
|
+
pe_name: pe_name,
|
101
|
+
ppn: ppn,
|
102
|
+
ld_library_path: ld_library_path,
|
103
|
+
command: command
|
104
|
+
)
|
105
|
+
rescue self::InitializeError
|
106
|
+
puts "Not #{self} : #{dir}"
|
107
|
+
rescue Comana::ComputationManager::AlreadySubmittedError
|
108
|
+
puts "Already started: #{dir}"
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
## jobs < hosts のキューがあれば(空きホストがあれば)、その中で bench が最小のもの
|
114
|
+
## なければ、self.guess_end_time の値が最小のもの。
|
115
|
+
def self.effective_queue(queues, jobs, hosts, benchmarks)
|
116
|
+
candidates = queues.select do |q|
|
117
|
+
jobs[q] < hosts[q]
|
118
|
+
end
|
119
|
+
if candidates.empty?
|
120
|
+
result = queues.min_by{|q| self.guess_end_time(jobs[q], hosts[q], benchmarks[q]) }
|
121
|
+
else
|
122
|
+
result = candidates.min_by {|q| benchmarks[q]}
|
123
|
+
end
|
124
|
+
result
|
125
|
+
end
|
126
|
+
|
127
|
+
# 新しく1個ジョブを追加した場合の終了見込み時間
|
128
|
+
# ジョブの終了時刻がランダムであり、ジョブの実行時間は等しいと仮定して算出。
|
129
|
+
# 空きホストがあれば benchmark 通りの見込み。
|
130
|
+
# 空きホストがなければ、ホストがあくまでの見込み時間を加算。
|
131
|
+
def self.guess_end_time(num_jobs , num_hosts, benchmark)
|
132
|
+
if num_jobs < num_hosts
|
133
|
+
unit = 1.0 #unit
|
134
|
+
else
|
135
|
+
unit = (num_jobs.to_f + 1.0) / (num_hosts.to_f)
|
136
|
+
end
|
137
|
+
result = unit * benchmark
|
138
|
+
result
|
139
|
+
end
|
140
|
+
|
43
141
|
# Return a symbol which indicate state of calculation.
|
44
142
|
# :yet not started
|
45
143
|
# :started started, but not ended, including short time from last output
|
@@ -55,7 +153,8 @@ class Comana::ComputationManager
|
|
55
153
|
# Execute calculation.
|
56
154
|
# If log of ComputationManager exist, raise ComputationManager::AlreadyStartedError,
|
57
155
|
# because the calculation has been done by other process already.
|
58
|
-
|
156
|
+
# This method is aliased to 'start'.
|
157
|
+
def execute
|
59
158
|
begin
|
60
159
|
Dir.mkdir "#{@dir}/#{@lockdir}"
|
61
160
|
rescue Errno::EEXIST
|
@@ -68,7 +167,7 @@ class Comana::ComputationManager
|
|
68
167
|
prepare_next
|
69
168
|
end
|
70
169
|
end
|
71
|
-
alias start
|
170
|
+
alias start execute
|
72
171
|
|
73
172
|
# Return latest modified time of files in calc dir recursively.
|
74
173
|
# require "find"
|
@@ -79,6 +178,31 @@ class Comana::ComputationManager
|
|
79
178
|
File.mtime(tmp)
|
80
179
|
end
|
81
180
|
|
181
|
+
def queue_submit(q_name:, pe_name:, ppn:, ld_library_path: , command:, submit: true)
|
182
|
+
|
183
|
+
qsub_path = "#{@dir}/#{QSUB_SCRIPT_NAME}"
|
184
|
+
if FileTest.exist? qsub_path
|
185
|
+
raise AlreadySubmittedError,
|
186
|
+
"Already exist #{qsub_path}."
|
187
|
+
end
|
188
|
+
File.open(qsub_path, "w") do |io|
|
189
|
+
Comana::GridEngine.write_qsub_script(
|
190
|
+
q_name: q_name,
|
191
|
+
pe_name: pe_name,
|
192
|
+
ppn: ppn,
|
193
|
+
ld_library_path: ld_library_path,
|
194
|
+
command: command,
|
195
|
+
io: io
|
196
|
+
)
|
197
|
+
end
|
198
|
+
cur_dir = Dir.pwd
|
199
|
+
Dir.chdir @dir
|
200
|
+
print "Submitting #{qsub_path}..."
|
201
|
+
system("qsub #{QSUB_SCRIPT_NAME} > #{QSUB_LOG_NAME}") if submit
|
202
|
+
puts "Done."
|
203
|
+
Dir.chdir cur_dir
|
204
|
+
end
|
205
|
+
|
82
206
|
private
|
83
207
|
|
84
208
|
# Redefine in subclass, e.g.,
|