osc-machete 1.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,72 @@
1
+ # Class that maintains the name and home identifiers of a User.
2
+ # Helper methods provided use the Etc module underneath.
3
+ #
4
+ class OSC::Machete::User
5
+
6
+ attr_reader :name
7
+
8
+ # default user is the username of the current process
9
+ #
10
+ # FIXME: is this true? Etc.getpwuid claims to use the default
11
+ # value from the Passwd struct:
12
+ # http://docs.ruby-lang.org/en/2.0.0/Etc.html#Passwd
13
+ # could this ever be different from Process.gid?
14
+ # Should we provide constructors for a User object for the given uid
15
+ # instead of username, or in Process do
16
+ # OSC::Machete::User.new(Etc.getpwuid.name(Process.gid))
17
+ # Or should the default be OSC::Machete::User.from_uid(Process.uid)
18
+ # Is there ever a difference between the two?
19
+ #
20
+ def initialize(username = Etc.getpwuid.name)
21
+ @name = username
22
+ end
23
+
24
+ # factory method to produce a User from specified uid
25
+ #
26
+ # @return [User] user for the specified uid
27
+ def self.from_uid(uid)
28
+ self.new Etc.getpwuid(uid).name
29
+ end
30
+
31
+ # Determine if user is member of specified group
32
+ #
33
+ # @param [String] group name
34
+ # @return [Boolean] true if user is a member of the specified group
35
+ def member_of_group?(group)
36
+ Etc.getgrnam(group).mem.include?(@name) rescue false
37
+ end
38
+
39
+ # get sorted list of group ids that user is part of
40
+ # by inspecting the /etc/group file
41
+ # there is also a ruby impl of this
42
+ #
43
+ # @return [Array] ids of groups that the user is a member of
44
+ def groups
45
+ `id -G $USER`.strip.split.map(&:to_i).uniq.sort
46
+ end
47
+
48
+ # get list of projects the user is part of
49
+ # FIXME: OSC specific
50
+ #
51
+ # @return [Array<String>] of projects the user is part of
52
+ def projects
53
+ `id -Gn`.split.grep(/^P./)
54
+ end
55
+
56
+ # FIXME: should we be using Pathnames here?
57
+ #
58
+ # Return Pathname for home directory
59
+ # The home directory path of the user.
60
+ #
61
+ # @return [Pathname] The directory path.
62
+ # def home
63
+ # Pathname.new(Dir.home(@name))
64
+ # end
65
+
66
+ # The home directory path of the user.
67
+ #
68
+ # @return [String] path to the home directory.
69
+ def home
70
+ Dir.home(@name)
71
+ end
72
+ end
@@ -0,0 +1,6 @@
1
+ module OSC
2
+ module Machete
3
+ # The current gem version
4
+ VERSION = "1.1.3"
5
+ end
6
+ end
@@ -0,0 +1,30 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'osc/machete/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "osc-machete"
8
+ spec.version = OSC::Machete::VERSION
9
+ spec.platform = Gem::Platform::RUBY
10
+ spec.authors = ["Eric Franz"]
11
+ spec.email = ["efranz@osc.edu"]
12
+ spec.summary = "Common interface for working with HPC batch jobs (currently OSC specific)"
13
+ spec.description = "Common interface for PBS (and eventually other resource managers and batch schedulers - currently OSC specific)"
14
+ spec.homepage = "https://github.com/OSC/osc-machete"
15
+ spec.license = "MIT"
16
+
17
+ spec.files = `git ls-files`.split($/)
18
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
19
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
20
+ spec.require_paths = ["lib"]
21
+
22
+ spec.add_development_dependency "bundler", "~> 1.3"
23
+ spec.add_development_dependency "rake"
24
+ spec.add_development_dependency "mocha"
25
+ spec.add_development_dependency "minitest", ">= 5.0"
26
+
27
+ spec.add_runtime_dependency "mustache"
28
+ spec.add_runtime_dependency "pbs", "~> 1.1"
29
+ end
30
+
@@ -0,0 +1,8 @@
1
+ ---
2
+ geometry: 'test.stl'
3
+ walltime: '00:30:00'
4
+ nodes: 'nodes=1:ppn=12'
5
+ job_name: 'foobar'
6
+ initial_temp: 80
7
+ density: 0.0027
8
+ coef: 1
@@ -0,0 +1,40 @@
1
+ #PBS -l walltime=00:30:00
2
+ #PBS -l nodes=1:ppn=12
3
+ #PBS -N foobar
4
+ #PBS -j oe
5
+ #PBS -r n
6
+
7
+ echo ----
8
+ echo Job started at `date`
9
+ echo ----
10
+ echo This job is working on compute node `cat $PBS_NODEFILE`
11
+
12
+ echo "TEMP IS 80"
13
+
14
+ cd $PBS_O_WORKDIR
15
+ echo show what PBS_O_WORKDIR is
16
+ echo PBS_O_WORKDIR IS `pwd`
17
+ echo ----
18
+ echo The contents of PBS_O_WORKDIR:
19
+ ls -ltr
20
+ echo
21
+ echo ----
22
+ echo
23
+ echo creating a file in PBS_O_WORKDIR
24
+ whoami > whoami-pbs-o-workdir
25
+
26
+ cd $TMPDIR
27
+ echo ----
28
+ echo TMPDIR IS `pwd`
29
+ echo ----
30
+ echo wait for 42 seconds
31
+ sleep 42
32
+ echo ----
33
+ echo creating a file in TMPDIR
34
+ whoami > whoami-tmpdir
35
+
36
+ # copy the file back to the output subdirectory
37
+ pbsdcp -g $TMPDIR/whoami-tmpdir $PBS_O_WORKDIR/output
38
+
39
+ echo ----
40
+ echo Job ended at `date`
@@ -0,0 +1,8 @@
1
+ ---
2
+ geometry: 'test.stl'
3
+ walltime: '00:30:00'
4
+ nodes: 'nodes=1:ppn=12'
5
+ job_name: 'foobar'
6
+ initial_temp: 80
7
+ density: 0.0027
8
+ coef: 1
@@ -0,0 +1,40 @@
1
+ #PBS -l walltime=00:30:00
2
+ #PBS -l nodes=1:ppn=12
3
+ #PBS -N foobar
4
+ #PBS -j oe
5
+ #PBS -r n
6
+
7
+ echo ----
8
+ echo Job started at `date`
9
+ echo ----
10
+ echo This job is working on compute node `cat $PBS_NODEFILE`
11
+
12
+ echo "TEMP IS 80"
13
+
14
+ cd $PBS_O_WORKDIR
15
+ echo show what PBS_O_WORKDIR is
16
+ echo PBS_O_WORKDIR IS `pwd`
17
+ echo ----
18
+ echo The contents of PBS_O_WORKDIR:
19
+ ls -ltr
20
+ echo
21
+ echo ----
22
+ echo
23
+ echo creating a file in PBS_O_WORKDIR
24
+ whoami > whoami-pbs-o-workdir
25
+
26
+ cd $TMPDIR
27
+ echo ----
28
+ echo TMPDIR IS `pwd`
29
+ echo ----
30
+ echo wait for 42 seconds
31
+ sleep 42
32
+ echo ----
33
+ echo creating a file in TMPDIR
34
+ whoami > whoami-tmpdir
35
+
36
+ # copy the file back to the output subdirectory
37
+ pbsdcp -g $TMPDIR/whoami-tmpdir $PBS_O_WORKDIR/output
38
+
39
+ echo ----
40
+ echo Job ended at `date`
@@ -0,0 +1,40 @@
1
+ #PBS -l walltime={{walltime}}
2
+ #PBS -l {{nodes}}
3
+ #PBS -N {{job_name}}
4
+ #PBS -j oe
5
+ #PBS -r n
6
+
7
+ echo ----
8
+ echo Job started at `date`
9
+ echo ----
10
+ echo This job is working on compute node `cat $PBS_NODEFILE`
11
+
12
+ echo "TEMP IS {{initial_temp}}"
13
+
14
+ cd $PBS_O_WORKDIR
15
+ echo show what PBS_O_WORKDIR is
16
+ echo PBS_O_WORKDIR IS `pwd`
17
+ echo ----
18
+ echo The contents of PBS_O_WORKDIR:
19
+ ls -ltr
20
+ echo
21
+ echo ----
22
+ echo
23
+ echo creating a file in PBS_O_WORKDIR
24
+ whoami > whoami-pbs-o-workdir
25
+
26
+ cd $TMPDIR
27
+ echo ----
28
+ echo TMPDIR IS `pwd`
29
+ echo ----
30
+ echo wait for 42 seconds
31
+ sleep 42
32
+ echo ----
33
+ echo creating a file in TMPDIR
34
+ whoami > whoami-tmpdir
35
+
36
+ # copy the file back to the output subdirectory
37
+ pbsdcp -g $TMPDIR/whoami-tmpdir $PBS_O_WORKDIR/output
38
+
39
+ echo ----
40
+ echo Job ended at `date`
@@ -0,0 +1,8 @@
1
+ ---
2
+ geometry: '{{geometry}}'
3
+ walltime: '{{walltime}}'
4
+ nodes: '{{nodes}}'
5
+ job_name: '{{job_name}}'
6
+ initial_temp: {{initial_temp}}
7
+ density: {{density}}
8
+ coef: {{coef}}
@@ -0,0 +1,40 @@
1
+ #PBS -l walltime={{walltime}}
2
+ #PBS -l {{nodes}}
3
+ #PBS -N {{job_name}}
4
+ #PBS -j oe
5
+ #PBS -r n
6
+
7
+ echo ----
8
+ echo Job started at `date`
9
+ echo ----
10
+ echo This job is working on compute node `cat $PBS_NODEFILE`
11
+
12
+ echo "TEMP IS {{initial_temp}}"
13
+
14
+ cd $PBS_O_WORKDIR
15
+ echo show what PBS_O_WORKDIR is
16
+ echo PBS_O_WORKDIR IS `pwd`
17
+ echo ----
18
+ echo The contents of PBS_O_WORKDIR:
19
+ ls -ltr
20
+ echo
21
+ echo ----
22
+ echo
23
+ echo creating a file in PBS_O_WORKDIR
24
+ whoami > whoami-pbs-o-workdir
25
+
26
+ cd $TMPDIR
27
+ echo ----
28
+ echo TMPDIR IS `pwd`
29
+ echo ----
30
+ echo wait for 42 seconds
31
+ sleep 42
32
+ echo ----
33
+ echo creating a file in TMPDIR
34
+ whoami > whoami-tmpdir
35
+
36
+ # copy the file back to the output subdirectory
37
+ pbsdcp -g $TMPDIR/whoami-tmpdir $PBS_O_WORKDIR/output
38
+
39
+ echo ----
40
+ echo Job ended at `date`
@@ -0,0 +1,14 @@
1
+ #PBS -l walltime=00:30:00
2
+ #PBS -l nodes=1:ppn=12
3
+ #PBS -S /bin/bash
4
+ #PBS -q @oak-batch.osc.edu
5
+ #PBS -N foobar
6
+ #PBS -j oe
7
+ #PBS -r n
8
+
9
+ echo ----
10
+ echo Job started at `date`
11
+ echo ----
12
+ echo This job is working on compute node `cat $PBS_NODEFILE`
13
+
14
+ echo "TEMP IS 80"
@@ -0,0 +1,14 @@
1
+ #PBS -l walltime=00:30:00
2
+ #PBS -l nodes=1:ppn=12
3
+ #PBS -S /bin/bash
4
+ #PBS -q @quick-batch.osc.edu
5
+ #PBS -N foobar
6
+ #PBS -j oe
7
+ #PBS -r n
8
+
9
+ echo ----
10
+ echo Job started at `date`
11
+ echo ----
12
+ echo This job is working on compute node `cat $PBS_NODEFILE`
13
+
14
+ echo "TEMP IS 80"
@@ -0,0 +1,14 @@
1
+ #PBS -l walltime=00:30:00
2
+ #PBS -l nodes=1:ppn=20
3
+ #PBS -S /bin/bash
4
+ #PBS -q @ruby-batch.osc.edu
5
+ #PBS -N foobar
6
+ #PBS -j oe
7
+ #PBS -r n
8
+
9
+ echo ----
10
+ echo Job started at `date`
11
+ echo ----
12
+ echo This job is working on compute node `cat $PBS_NODEFILE`
13
+
14
+ echo "TEMP IS 80"
data/test/test_job.rb ADDED
@@ -0,0 +1,179 @@
1
+ require 'minitest/autorun'
2
+ require 'osc/machete'
3
+
4
+ class TestJob < Minitest::Test
5
+ def setup
6
+ @id1 = "16376371.opt-batch.osc.edu"
7
+ @id2 = "16376372.opt-batch.osc.edu"
8
+
9
+ @jobdir = Pathname.new(Dir.mktmpdir)
10
+ @scriptname = "main.sh"
11
+ @scriptpath = @jobdir.join(@scriptname)
12
+ FileUtils.touch(@scriptpath)
13
+ end
14
+
15
+ def teardown
16
+ @jobdir.rmtree if @jobdir.exist?
17
+ end
18
+
19
+ def test_basic_job
20
+ job = OSC::Machete::Job.new(script: @scriptpath)
21
+ assert_equal job.path.to_s, @jobdir.to_s
22
+ assert_equal job.script_name, @scriptname
23
+ end
24
+
25
+ # test outgoing qsub messages send correct dependency arguments
26
+ # when setting up a job that depends on another job
27
+ def test_job_dependency_afterany
28
+ # create first job and expect qsub to work as it does before
29
+ torque1 = OSC::Machete::TorqueHelper.new
30
+ torque1.expects(:qsub).with(@scriptname, has_entry(depends_on: {})).returns(@id1)
31
+ job1 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque1
32
+
33
+ # create second job that depends on the first and expect qsub to send pbsid of the first job
34
+ # which will not be known until the first job qsub-ed and 16376371.opt-batch.osc.edu is returned
35
+ torque2 = OSC::Machete::TorqueHelper.new
36
+ torque2.expects(:qsub).with(@scriptname, has_entry(depends_on: { afterany: [@id1] })).returns(@id2)
37
+ job2 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque2
38
+
39
+ # add the dependency and submit the job
40
+ job2.afterany job1
41
+ job2.submit
42
+
43
+ # assertions
44
+ assert job1.submitted?, "dependent job not submitted"
45
+ assert job2.submitted?, "job not submitted"
46
+
47
+ assert_equal @id1, job1.pbsid
48
+ assert_equal @id2, job2.pbsid
49
+ end
50
+
51
+ # test outgoing qsub messages send correct dependency arguments
52
+ # when setting up a job that depends on another job
53
+ def test_job_dependency_afterok
54
+ # create first job and expect qsub to work as it does before
55
+ torque1 = OSC::Machete::TorqueHelper.new
56
+ torque1.expects(:qsub).with(@scriptname, has_entry(depends_on: {})).returns(@id1)
57
+ job1 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque1
58
+
59
+ # create second job that depends on the first and expect qsub to send pbsid of the first job
60
+ # which will not be known until the first job qsub-ed and 16376371.opt-batch.osc.edu is returned
61
+ torque2 = OSC::Machete::TorqueHelper.new
62
+ torque2.expects(:qsub).with(@scriptname, has_entry(depends_on: { afterok: [@id1] })).returns(@id2)
63
+ job2 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque2
64
+
65
+ # add the dependency and submit the job
66
+ job2.afterok job1
67
+ job2.submit
68
+
69
+ # assertions
70
+ assert job1.submitted?, "dependent job not submitted"
71
+ assert job2.submitted?, "job not submitted"
72
+
73
+ assert_equal @id1, job1.pbsid
74
+ assert_equal @id2, job2.pbsid
75
+ end
76
+
77
+ # here we repeat the above test but when setting up the dependency we chain
78
+ # OSC::Machete::Job.new(...).afterok(...)
79
+ # as long as afterok returns self, chaining will work
80
+ def test_job_dependency_afterok_chaining
81
+ # create first job and expect qsub to work as it does before
82
+ torque1 = OSC::Machete::TorqueHelper.new
83
+ torque1.expects(:qsub).with(@scriptname, has_entry(depends_on: {})).returns(@id1)
84
+ job1 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque1
85
+
86
+ # create second job that depends on the first and expect qsub to send pbsid of the first job
87
+ # which will not be known until the first job qsub-ed and 16376371.opt-batch.osc.edu is returned
88
+ torque2 = OSC::Machete::TorqueHelper.new
89
+ torque2.expects(:qsub).with(@scriptname, has_entry(depends_on: { afterok: [@id1] })).returns(@id2)
90
+ job2 = OSC::Machete::Job.new(script: @scriptpath, torque_helper: torque2).afterok(job1)
91
+ job2.submit
92
+ end
93
+
94
+ def test_job_dependency_after
95
+ # create first job and expect qsub to work as it does before
96
+ torque1 = OSC::Machete::TorqueHelper.new
97
+ torque1.expects(:qsub).with(@scriptname, has_entry(depends_on: {})).returns(@id1)
98
+ job1 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque1
99
+
100
+ # create second job that depends on the first and expect qsub to send pbsid of the first job
101
+ # which will not be known until the first job qsub-ed and 16376371.opt-batch.osc.edu is returned
102
+ torque2 = OSC::Machete::TorqueHelper.new
103
+ torque2.expects(:qsub).with(@scriptname, has_entry(depends_on: { after: [@id1] })).returns(@id2)
104
+ job2 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque2
105
+
106
+ # add the dependency and submit the job
107
+ job2.after job1
108
+ job2.submit
109
+ end
110
+
111
+ def test_job_status
112
+ torque1 = OSC::Machete::TorqueHelper.new
113
+ torque1.expects(:qstat).with(@id1, any_parameters).returns(OSC::Machete::Status.running)
114
+ job = OSC::Machete::Job.new pbsid: @id1, torque_helper: torque1
115
+
116
+ assert_equal job.status, OSC::Machete::Status.running
117
+
118
+ job = OSC::Machete::Job.new script: @scriptpath
119
+ assert_equal job.status, OSC::Machete::Status.not_submitted
120
+ end
121
+
122
+
123
+ def test_job_delete
124
+ # FIXME: the unit tests in this file are not dry...
125
+ # FIXME: rethink the interface: should delete return true if a job was actually deleted?
126
+
127
+ torque1 = OSC::Machete::TorqueHelper.new
128
+ torque1.expects(:qdel).with(@id1, any_parameters).returns(true)
129
+ job = OSC::Machete::Job.new(script: @scriptpath, pbsid: @id1, torque_helper: torque1)
130
+ job.delete
131
+ assert @jobdir.exist?, "deleting job by default should not deleted the directory too"
132
+
133
+ torque2 = OSC::Machete::TorqueHelper.new
134
+ torque2.expects(:qdel).with(@id1, any_parameters).returns(true)
135
+ job = OSC::Machete::Job.new(script: @scriptpath, pbsid: @id1, torque_helper: torque2)
136
+ job.delete(rmdir: true)
137
+
138
+ assert ! @jobdir.exist?, "deleting job and specifying rmdir:true should have deleted the directory too"
139
+ end
140
+
141
+ def test_job_dependency_delete
142
+ torque1 = OSC::Machete::TorqueHelper.new
143
+ torque1.expects(:qdel).with(@id1, any_parameters).returns(true)
144
+ torque1.expects(:qdel).with(@id2, any_parameters).returns(true)
145
+ job1 = OSC::Machete::Job.new(script: @scriptpath, pbsid: @id1, torque_helper: torque1)
146
+ job2 = OSC::Machete::Job.new(script: @scriptpath, pbsid: @id2, torque_helper: torque1)
147
+
148
+ job2.afterok job1
149
+
150
+ # we want to be able to safely delete two jobs that share the same pbs_work_dir and don't
151
+ # want to be able to call rmdir: true on both without worrying that the first one deleted
152
+ # the actual directory so the second might fail
153
+ # both should succeed (if the directory doesn't exist, just ignore that step)
154
+ job1.delete(rmdir: true)
155
+ job2.delete(rmdir: true)
156
+ end
157
+
158
+
159
+ def assert_submit_qsubs_with_account_string(args, account_string: nil)
160
+ # create first job and expect qsub to work as it does before
161
+ torque1 = OSC::Machete::TorqueHelper.new
162
+ torque1.expects(:qsub).with(@scriptname, has_entry(account_string: account_string)).returns(@id1)
163
+ OSC::Machete::Job.new(args.merge({script: @scriptpath, torque_helper: torque1})).submit
164
+ end
165
+
166
+ def test_account_string_not_set
167
+ # with default to nil we ensure that if we pass in account_string to
168
+ # initializer its used in qsub
169
+ assert_submit_qsubs_with_account_string({}, account_string: nil)
170
+ assert_submit_qsubs_with_account_string({account_string: "PZ3"}, account_string: "PZ3")
171
+
172
+ # if default set, we use default unless account_string is passed
173
+ OSC::Machete::Job.default_account_string = "FOO"
174
+ assert_submit_qsubs_with_account_string({account_string: "PZ3"}, account_string: "PZ3")
175
+ assert_submit_qsubs_with_account_string({}, account_string: "FOO")
176
+
177
+ OSC::Machete::Job.default_account_string = nil
178
+ end
179
+ end