pbs_job 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/pbs_job +1 -172
- data/lib/pbs_job/version.rb +1 -1
- data/lib/pbs_job.rb +157 -1
- data/spec/pbs_job_spec.rb +49 -18
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 39763ec22c9bc359927319d9cf3292c0846ff189
|
4
|
+
data.tar.gz: e434e73ffae3ab0c3a0501942159f1a6b65ac680
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7ee324e95e9275a6192ad6b5c1be77ace1a945c70ea2bb865f9468a1d6cb3e8c3fc09f9bbe4b7b16ff2683a9b456bdc0962d49715f18d44e6c81cc1c45209aa7
|
7
|
+
data.tar.gz: 4474e85e12cf8546411f3482e169e8a124bbeceb3e7de39352845414601c57657e1933e36449e5204a50663675ba1a5f400027e289e1669be7635f42e9fc45f2
|
data/bin/pbs_job
CHANGED
@@ -1,178 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
require "rubygems"
|
3
|
-
require 'thor'
|
4
3
|
require 'date'
|
5
4
|
|
6
|
-
|
7
|
-
class New < Thor::Group
|
8
|
-
include Thor::Actions
|
9
|
-
|
10
|
-
STREAMS_DIR_NAME = 'streams'
|
11
|
-
EXECUTABLE_PERMISSIONS = 0775
|
12
|
-
|
13
|
-
argument :name, :desc => 'Name of the new job.'
|
14
|
-
argument :email_address, :desc => 'Email to which to send PBS alerts.'
|
15
|
-
|
16
|
-
class_option(
|
17
|
-
:link_results,
|
18
|
-
{
|
19
|
-
:desc => 'Directory to which to redirect results',
|
20
|
-
:default => nil,
|
21
|
-
:aliases => 'r'
|
22
|
-
}
|
23
|
-
)
|
24
|
-
class_option(
|
25
|
-
:script,
|
26
|
-
{
|
27
|
-
:desc => 'Type of script to make the task script',
|
28
|
-
:default => 'bash',
|
29
|
-
:aliases => 's'
|
30
|
-
}
|
31
|
-
)
|
32
|
-
class_option(
|
33
|
-
:task_working_dir,
|
34
|
-
{
|
35
|
-
:desc => 'Working directory in which to run task',
|
36
|
-
:default => File.expand_path('~'),
|
37
|
-
:aliases => 'w'
|
38
|
-
}
|
39
|
-
)
|
40
|
-
|
41
|
-
class_option(
|
42
|
-
:walltime,
|
43
|
-
{
|
44
|
-
:desc => '#PBS -l walltime=72:00:00
|
45
|
-
This is the maximum elapsed time that the job will be allowed to run, specified as hours, minutes and seconds in the form HH:MM:SS. If a job exceeds its walltime limit, it is killed by the system. It is best to overestimate the walltime to avoid a run being spoiled by early termination, however, an accurate walltime estimate will allow your job to be scheduled more effectively. It is best to design your code with a capability to write checkpoint data to a file periodically and to be able to restart from the time of the most recent checkpoint by reading that data. That way if the run reaches its walltime limit (or fails for some other reason), only a small fraction of the total computation will have to be redone in a subsequent run. (https://www.westgrid.ca/support/running_jobs)',
|
46
|
-
:default => nil
|
47
|
-
}
|
48
|
-
)
|
49
|
-
class_option(
|
50
|
-
:mem,
|
51
|
-
{
|
52
|
-
:desc => '#PBS -l mem=2000mb
|
53
|
-
The mem parameter should be an estimate of the total amount of memory required by the job. For parallel jobs, multiply the memory per process by the number of processes. Append units of MB or GB as appropriate. The value given must be an integer, so, for example, use mem=3584MB instead of mem=3.5GB (1 GB = 1024 MB) .
|
54
|
-
Note: the mem parameter is not used on the large shared-memory machines. . . . (https://www.westgrid.ca/support/running_jobs)',
|
55
|
-
:default => nil
|
56
|
-
}
|
57
|
-
)
|
58
|
-
class_option(
|
59
|
-
:nodes,
|
60
|
-
{
|
61
|
-
:desc => '#PBS -l nodes=4:ppn=2
|
62
|
-
Use a combination of nodes and processors per node (ppn) to request the total number of processors needed. (https://www.westgrid.ca/support/running_jobs)',
|
63
|
-
:default => nil
|
64
|
-
}
|
65
|
-
)
|
66
|
-
class_option(
|
67
|
-
:ppn,
|
68
|
-
{
|
69
|
-
:desc => '#PBS -l nodes=4:ppn=2
|
70
|
-
Use a combination of nodes and processors per node (ppn) to request the total number of processors needed. (https://www.westgrid.ca/support/running_jobs)',
|
71
|
-
:default => 1
|
72
|
-
}
|
73
|
-
)
|
74
|
-
class_option(
|
75
|
-
:pmem,
|
76
|
-
{
|
77
|
-
:desc => '#PBS -l pmem=2000mb
|
78
|
-
Instead of specifying the total memory requirement of your job with the mem parameter . . ., you can specify a per process memory limit, pmem. Note however, that mem and pmem are independent parameters, so, on some systems it may be necessary to specify both mem and pmem. (https://www.westgrid.ca/support/running_jobs)',
|
79
|
-
:default => nil
|
80
|
-
}
|
81
|
-
)
|
82
|
-
class_option(
|
83
|
-
:procs,
|
84
|
-
{
|
85
|
-
:desc => '#PBS -l procs=8
|
86
|
-
The procs resource request allows the scheduler to distribute the requested number of processors among any available nodes on the cluster. This can reduce the waiting time in the input queue. . . .
|
87
|
-
In using this resource request format, you are not guaranteed any specific number of processors per node. As such, it is not appropriate for OpenMP programs or other multi-threaded programs, with rare exception. For example, if you have a mixed MPI-OpenMP program in which you limit the number of threads per process to just one (OMP_NUM_THREADS=1) you could use procs. In other cases, you may also have to combine procs with pmem to make sure that the scheduler does not assign too many of your processes to the same node. (https://www.westgrid.ca/support/running_jobs)',
|
88
|
-
:default => nil
|
89
|
-
}
|
90
|
-
)
|
91
|
-
|
92
|
-
def self.source_root
|
93
|
-
File.expand_path('../../', __FILE__)
|
94
|
-
end
|
95
|
-
|
96
|
-
def gen_root
|
97
|
-
empty_directory full_name
|
98
|
-
end
|
99
|
-
|
100
|
-
def qsub_script
|
101
|
-
create_script_from_template 'job.qsub'
|
102
|
-
end
|
103
|
-
|
104
|
-
def pbs_script
|
105
|
-
create_script_from_template 'job.pbs'
|
106
|
-
end
|
107
|
-
|
108
|
-
def task
|
109
|
-
create_script_from_template 'task'
|
110
|
-
end
|
111
|
-
|
112
|
-
def results
|
113
|
-
results_path = File.join full_name, 'results'
|
114
|
-
if options[:link_results]
|
115
|
-
create_link(results_path, options[:link_results])
|
116
|
-
else
|
117
|
-
empty_directory results_path
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
def streams
|
122
|
-
empty_directory streams_path
|
123
|
-
end
|
124
|
-
|
125
|
-
private
|
126
|
-
|
127
|
-
def file_path(script_name)
|
128
|
-
File.join full_name, script_name
|
129
|
-
end
|
130
|
-
|
131
|
-
def create_file_from_template(file_name)
|
132
|
-
template(
|
133
|
-
"templates/#{file_name}.tt",
|
134
|
-
file_path(file_name)
|
135
|
-
)
|
136
|
-
end
|
137
|
-
|
138
|
-
def create_script_from_template(script_name)
|
139
|
-
create_file_from_template script_name
|
140
|
-
chmod file_path(script_name), EXECUTABLE_PERMISSIONS
|
141
|
-
end
|
142
|
-
|
143
|
-
# @returns [String] Name with date appended
|
144
|
-
def full_name
|
145
|
-
@full_name ||= lambda do
|
146
|
-
today = Date.today
|
147
|
-
month = Date::MONTHNAMES[today.month].downcase
|
148
|
-
"#{name}.#{month}#{today.day}_#{today.year}"
|
149
|
-
end.call
|
150
|
-
end
|
151
|
-
|
152
|
-
def abs_job_root
|
153
|
-
@abs_job_root ||= File.expand_path(full_name)
|
154
|
-
end
|
155
|
-
|
156
|
-
def streams_path
|
157
|
-
@streams_path ||= File.join full_name, STREAMS_DIR_NAME
|
158
|
-
end
|
159
|
-
|
160
|
-
def abs_stream_prefix
|
161
|
-
@abs_stream_prefix ||= File.join abs_job_root, STREAMS_DIR_NAME, full_name
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
class PbsJob < Thor
|
166
|
-
map 'n' => :new
|
167
|
-
|
168
|
-
register(
|
169
|
-
New,
|
170
|
-
'new',
|
171
|
-
'new NAME EMAIL_ADDRESS [OPTIONS]',
|
172
|
-
"Creates a new PBS job with the name NAME and arranges for PBS alerts to be sent to EMAIL_ADDRESS, customized by OPTIONS"
|
173
|
-
)
|
174
|
-
tasks["new"].options = New.class_options
|
175
|
-
end
|
176
|
-
end
|
5
|
+
require 'pbs_job'
|
177
6
|
|
178
7
|
PbsJob::PbsJob.start(ARGV)
|
data/lib/pbs_job/version.rb
CHANGED
data/lib/pbs_job.rb
CHANGED
@@ -1,4 +1,160 @@
|
|
1
1
|
require_relative "pbs_job/version"
|
2
2
|
|
3
|
-
|
3
|
+
require 'thor'
|
4
|
+
|
5
|
+
module PbsJob
|
6
|
+
class New < Thor::Group
|
7
|
+
include Thor::Actions
|
8
|
+
|
9
|
+
STREAMS_DIR_NAME = 'streams'
|
10
|
+
EXECUTABLE_PERMISSIONS = 0775
|
11
|
+
|
12
|
+
argument :name, :desc => 'Name of the new job.'
|
13
|
+
argument :email_address, :desc => 'Email to which to send PBS alerts.'
|
14
|
+
|
15
|
+
class_option(
|
16
|
+
:link_results,
|
17
|
+
{
|
18
|
+
:desc => 'Directory to which to redirect results',
|
19
|
+
:default => nil,
|
20
|
+
:aliases => 'r'
|
21
|
+
}
|
22
|
+
)
|
23
|
+
class_option(
|
24
|
+
:script,
|
25
|
+
{
|
26
|
+
:desc => 'Type of script to make the task script',
|
27
|
+
:default => 'bash',
|
28
|
+
:aliases => 's'
|
29
|
+
}
|
30
|
+
)
|
31
|
+
class_option(
|
32
|
+
:task_working_dir,
|
33
|
+
{
|
34
|
+
:desc => 'Working directory in which to run task',
|
35
|
+
:default => File.expand_path('~'),
|
36
|
+
:aliases => 'w'
|
37
|
+
}
|
38
|
+
)
|
39
|
+
|
40
|
+
class_option(
|
41
|
+
:walltime,
|
42
|
+
{
|
43
|
+
:desc => '#PBS -l walltime=72:00:00
|
44
|
+
This is the maximum elapsed time that the job will be allowed to run, specified as hours, minutes and seconds in the form HH:MM:SS. If a job exceeds its walltime limit, it is killed by the system. It is best to overestimate the walltime to avoid a run being spoiled by early termination, however, an accurate walltime estimate will allow your job to be scheduled more effectively. It is best to design your code with a capability to write checkpoint data to a file periodically and to be able to restart from the time of the most recent checkpoint by reading that data. That way if the run reaches its walltime limit (or fails for some other reason), only a small fraction of the total computation will have to be redone in a subsequent run. (https://www.westgrid.ca/support/running_jobs)',
|
45
|
+
:default => nil
|
46
|
+
}
|
47
|
+
)
|
48
|
+
class_option(
|
49
|
+
:mem,
|
50
|
+
{
|
51
|
+
:desc => '#PBS -l mem=2000mb
|
52
|
+
The mem parameter should be an estimate of the total amount of memory required by the job. For parallel jobs, multiply the memory per process by the number of processes. Append units of MB or GB as appropriate. The value given must be an integer, so, for example, use mem=3584MB instead of mem=3.5GB (1 GB = 1024 MB) .
|
53
|
+
Note: the mem parameter is not used on the large shared-memory machines. . . . (https://www.westgrid.ca/support/running_jobs)',
|
54
|
+
:default => nil
|
55
|
+
}
|
56
|
+
)
|
57
|
+
class_option(
|
58
|
+
:nodes,
|
59
|
+
{
|
60
|
+
:desc => '#PBS -l nodes=4:ppn=2
|
61
|
+
Use a combination of nodes and processors per node (ppn) to request the total number of processors needed. (https://www.westgrid.ca/support/running_jobs)',
|
62
|
+
:default => nil
|
63
|
+
}
|
64
|
+
)
|
65
|
+
class_option(
|
66
|
+
:ppn,
|
67
|
+
{
|
68
|
+
:desc => '#PBS -l nodes=4:ppn=2
|
69
|
+
Use a combination of nodes and processors per node (ppn) to request the total number of processors needed. (https://www.westgrid.ca/support/running_jobs)',
|
70
|
+
:default => 1
|
71
|
+
}
|
72
|
+
)
|
73
|
+
class_option(
|
74
|
+
:pmem,
|
75
|
+
{
|
76
|
+
:desc => '#PBS -l pmem=2000mb
|
77
|
+
Instead of specifying the total memory requirement of your job with the mem parameter . . ., you can specify a per process memory limit, pmem. Note however, that mem and pmem are independent parameters, so, on some systems it may be necessary to specify both mem and pmem. (https://www.westgrid.ca/support/running_jobs)',
|
78
|
+
:default => nil
|
79
|
+
}
|
80
|
+
)
|
81
|
+
class_option(
|
82
|
+
:procs,
|
83
|
+
{
|
84
|
+
:desc => '#PBS -l procs=8
|
85
|
+
The procs resource request allows the scheduler to distribute the requested number of processors among any available nodes on the cluster. This can reduce the waiting time in the input queue. . . .
|
86
|
+
In using this resource request format, you are not guaranteed any specific number of processors per node. As such, it is not appropriate for OpenMP programs or other multi-threaded programs, with rare exception. For example, if you have a mixed MPI-OpenMP program in which you limit the number of threads per process to just one (OMP_NUM_THREADS=1) you could use procs. In other cases, you may also have to combine procs with pmem to make sure that the scheduler does not assign too many of your processes to the same node. (https://www.westgrid.ca/support/running_jobs)',
|
87
|
+
:default => nil
|
88
|
+
}
|
89
|
+
)
|
90
|
+
|
91
|
+
def self.source_root
|
92
|
+
File.expand_path('../../', __FILE__)
|
93
|
+
end
|
94
|
+
|
95
|
+
def gen_root() empty_directory(full_name) end
|
96
|
+
def gen_qsub_script() create_script_from_template('job.qsub') end
|
97
|
+
def gen_pbs_script() create_script_from_template('job.pbs') end
|
98
|
+
def gen_task_script() create_script_from_template('task') end
|
99
|
+
def gen_streams_dir() empty_directory(streams_path) end
|
100
|
+
def gen_results_dir
|
101
|
+
results_path = File.join full_name, 'results'
|
102
|
+
if options[:link_results]
|
103
|
+
create_link(results_path, options[:link_results])
|
104
|
+
else
|
105
|
+
empty_directory results_path
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
private
|
110
|
+
|
111
|
+
def file_path(script_name)
|
112
|
+
File.join full_name, script_name
|
113
|
+
end
|
114
|
+
|
115
|
+
def create_file_from_template(file_name)
|
116
|
+
template(
|
117
|
+
"templates/#{file_name}.tt",
|
118
|
+
file_path(file_name)
|
119
|
+
)
|
120
|
+
end
|
121
|
+
|
122
|
+
def create_script_from_template(script_name)
|
123
|
+
create_file_from_template script_name
|
124
|
+
chmod file_path(script_name), EXECUTABLE_PERMISSIONS
|
125
|
+
end
|
126
|
+
|
127
|
+
# @returns [String] Name with date appended
|
128
|
+
def full_name
|
129
|
+
@full_name ||= lambda do
|
130
|
+
today = Date.today
|
131
|
+
month = Date::MONTHNAMES[today.month].downcase
|
132
|
+
"#{name}.#{month}#{today.day}_#{today.year}"
|
133
|
+
end.call
|
134
|
+
end
|
135
|
+
|
136
|
+
def abs_job_root
|
137
|
+
@abs_job_root ||= File.expand_path(full_name)
|
138
|
+
end
|
139
|
+
|
140
|
+
def streams_path
|
141
|
+
@streams_path ||= File.join full_name, STREAMS_DIR_NAME
|
142
|
+
end
|
143
|
+
|
144
|
+
def abs_stream_prefix
|
145
|
+
@abs_stream_prefix ||= File.join abs_job_root, STREAMS_DIR_NAME, full_name
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
class PbsJob < Thor
|
150
|
+
map 'n' => :new
|
151
|
+
|
152
|
+
register(
|
153
|
+
New,
|
154
|
+
'new',
|
155
|
+
'new NAME EMAIL_ADDRESS [OPTIONS]',
|
156
|
+
"Creates a new PBS job with the name NAME and arranges for PBS alerts to be sent to EMAIL_ADDRESS, customized by OPTIONS"
|
157
|
+
)
|
158
|
+
tasks["new"].options = New.class_options
|
159
|
+
end
|
4
160
|
end
|
data/spec/pbs_job_spec.rb
CHANGED
@@ -3,6 +3,19 @@ require_relative 'support/spec_helper'
|
|
3
3
|
require "date"
|
4
4
|
require 'tmpdir'
|
5
5
|
|
6
|
+
require_relative '../lib/pbs_job'
|
7
|
+
|
8
|
+
NAME = 'test_job'
|
9
|
+
EMAIL_ADDRESS = 'email@address'
|
10
|
+
TODAY = Date.today
|
11
|
+
MONTH = Date::MONTHNAMES[TODAY.month].downcase
|
12
|
+
JOB_ROOT = "#{NAME}.#{MONTH}#{TODAY.day}_#{TODAY.year}"
|
13
|
+
QSUB_SCRIPT = "#{JOB_ROOT}/job.qsub"
|
14
|
+
PBS_SCRIPT = "#{JOB_ROOT}/job.pbs"
|
15
|
+
TASK = "#{JOB_ROOT}/task"
|
16
|
+
RESULTS = "#{JOB_ROOT}/results"
|
17
|
+
STREAMS = "#{JOB_ROOT}/streams"
|
18
|
+
|
6
19
|
describe 'pbs_job' do
|
7
20
|
|
8
21
|
HELP = <<-HELP
|
@@ -11,16 +24,6 @@ Commands:
|
|
11
24
|
pbs_job new NAME EMAIL_ADDRESS [OPTIONS] # Creates a new PBS job with the ...
|
12
25
|
|
13
26
|
HELP
|
14
|
-
NAME = 'test_job'
|
15
|
-
EMAIL_ADDRESS = 'email@address'
|
16
|
-
TODAY = Date.today
|
17
|
-
MONTH = Date::MONTHNAMES[TODAY.month].downcase
|
18
|
-
JOB_ROOT = "#{NAME}.#{MONTH}#{TODAY.day}_#{TODAY.year}"
|
19
|
-
QSUB_SCRIPT = "#{JOB_ROOT}/job.qsub"
|
20
|
-
PBS_SCRIPT = "#{JOB_ROOT}/job.pbs"
|
21
|
-
TASK = "#{JOB_ROOT}/task"
|
22
|
-
RESULTS = "#{JOB_ROOT}/results"
|
23
|
-
STREAMS = "#{JOB_ROOT}/streams"
|
24
27
|
SUCCESSFUL_OUTPUT = <<-OUT
|
25
28
|
\e[1m\e[32m create\e[0m #{JOB_ROOT}
|
26
29
|
\e[1m\e[32m create\e[0m #{QSUB_SCRIPT}
|
@@ -29,8 +32,8 @@ HELP
|
|
29
32
|
\e[1m\e[32m chmod\e[0m #{PBS_SCRIPT}
|
30
33
|
\e[1m\e[32m create\e[0m #{TASK}
|
31
34
|
\e[1m\e[32m chmod\e[0m #{TASK}
|
32
|
-
\e[1m\e[32m create\e[0m #{RESULTS}
|
33
35
|
\e[1m\e[32m create\e[0m #{STREAMS}
|
36
|
+
\e[1m\e[32m create\e[0m #{RESULTS}
|
34
37
|
OUT
|
35
38
|
|
36
39
|
it 'prints usage and options without arguments' do
|
@@ -113,15 +116,43 @@ GEN_HELP
|
|
113
116
|
end
|
114
117
|
end
|
115
118
|
|
116
|
-
def in_tmp_dir
|
117
|
-
Dir.mktmpdir do |dir|
|
118
|
-
Dir.chdir dir
|
119
|
-
yield dir if block_given?
|
120
|
-
end
|
121
|
-
end
|
122
|
-
|
123
119
|
def check_script(script_path)
|
124
120
|
File.file?(script_path).must_equal true
|
125
121
|
File.executable?(script_path).must_equal true
|
126
122
|
end
|
127
123
|
end
|
124
|
+
|
125
|
+
describe PbsJob::PbsJob do
|
126
|
+
it '#start returns the job root directory as the first element of a list' do
|
127
|
+
args = ['new', NAME, EMAIL_ADDRESS]
|
128
|
+
in_tmp_dir do
|
129
|
+
patient = nil
|
130
|
+
quietly do
|
131
|
+
patient = File.basename(PbsJob::PbsJob.start(args).first)
|
132
|
+
end
|
133
|
+
patient.must_equal JOB_ROOT
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def quietly
|
139
|
+
$stdout.flush
|
140
|
+
$stderr.flush
|
141
|
+
previous_stdout = $stdout.dup
|
142
|
+
previous_stderr = $stderr.dup
|
143
|
+
begin
|
144
|
+
$stdout.reopen('/home/vagrant/temp/temp.out', 'w')
|
145
|
+
$stderr.reopen('/home/vagrant/temp/temp.err', 'w')
|
146
|
+
yield
|
147
|
+
ensure
|
148
|
+
$stdout.reopen(previous_stdout)
|
149
|
+
$stderr.reopen(previous_stderr)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
def in_tmp_dir
|
154
|
+
Dir.mktmpdir do |dir|
|
155
|
+
Dir.chdir dir
|
156
|
+
yield dir if block_given?
|
157
|
+
end
|
158
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pbs_job
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dustin Morrill
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-
|
11
|
+
date: 2013-12-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|