osc-machete 1.1.3

data/lib/osc/machete/user.rb

@@ -0,0 +1,72 @@
+# Class that maintains the name and home identifiers of a User.
+# Helper methods provided use the Etc module underneath.
+#
+class OSC::Machete::User
+
+  attr_reader :name
+
+  # default user is the username of the current process
+  #
+  # FIXME: is this true? Etc.getpwuid claims to use the default
+  # value from the Passwd struct:
+  # http://docs.ruby-lang.org/en/2.0.0/Etc.html#Passwd
+  # could this ever be different from Process.gid?
+  # Should we provide constructors for a User object for the given uid
+  # instead of username, or in Process do
+  # OSC::Machete::User.new(Etc.getpwuid.name(Process.gid))
+  # Or should the default be OSC::Machete::User.from_uid(Process.uid)
+  # Is there ever a difference between the two?
+  #
+  def initialize(username = Etc.getpwuid.name)
+    @name = username
+  end
+
+  # factory method to produce a User from specified uid
+  #
+  # @return [User] user for the specified uid
+  def self.from_uid(uid)
+    self.new Etc.getpwuid(uid).name
+  end
+
+  # Determine if user is member of specified group
+  #
+  # @param [String] group name
+  # @return [Boolean] true if user is a member of the specified group
+  def member_of_group?(group)
+    Etc.getgrnam(group).mem.include?(@name) rescue false
+  end
+
+  # get sorted list of group ids that user is part of
+  # by inspecting the /etc/group file
+  # there is also a ruby impl of this
+  #
+  # @return [Array] ids of groups that the user is a member of
+  def groups
+    `id -G $USER`.strip.split.map(&:to_i).uniq.sort
+  end
+
+  # get list of projects the user is part of
+  # FIXME: OSC specific
+  #
+  # @return [Array<String>] of projects the user is part of
+  def projects
+    `id -Gn`.split.grep(/^P./)
+  end
+
+  # FIXME: should we be using Pathnames here?
+  #
+  # Return Pathname for home directory
+  # The home directory path of the user.
+  #
+  # @return [Pathname] The directory path.
+  # def home
+  #   Pathname.new(Dir.home(@name))
+  # end
+
+  # The home directory path of the user.
+  #
+  # @return [String] path to the home directory.
+  def home
+    Dir.home(@name)
+  end
+end

data/lib/osc/machete/version.rb

@@ -0,0 +1,6 @@
+module OSC
+  module Machete
+    # The current gem version
+    VERSION = "1.1.3"
+  end
+end

data/osc-machete.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'osc/machete/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "osc-machete"
+  spec.version       = OSC::Machete::VERSION
+  spec.platform      = Gem::Platform::RUBY
+  spec.authors       = ["Eric Franz"]
+  spec.email         = ["efranz@osc.edu"]
+  spec.summary       = "Common interface for working with HPC batch jobs (currently OSC specific)"
+  spec.description   = "Common interface for PBS (and eventually other resource managers and batch schedulers - currently OSC specific)"
+  spec.homepage      = "https://github.com/OSC/osc-machete"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "mocha"
+  spec.add_development_dependency "minitest", ">= 5.0"
+
+  spec.add_runtime_dependency "mustache"
+  spec.add_runtime_dependency "pbs", "~> 1.1"
+end
+

data/test/fixtures/app-params.yml

@@ -0,0 +1,8 @@
+---
+geometry: 'test.stl'
+walltime: '00:30:00'
+nodes: 'nodes=1:ppn=12'
+job_name: 'foobar'
+initial_temp: 80
+density: 0.0027
+coef: 1

data/test/fixtures/app-template-rendered/job.sh

@@ -0,0 +1,40 @@
+#PBS -l walltime=00:30:00
+#PBS -l nodes=1:ppn=12
+#PBS -N foobar
+#PBS -j oe
+#PBS -r n
+
+echo ----
+echo Job started at `date`
+echo ----
+echo This job is working on compute node `cat $PBS_NODEFILE`
+
+echo "TEMP IS 80"
+
+cd $PBS_O_WORKDIR
+echo show what PBS_O_WORKDIR is
+echo PBS_O_WORKDIR IS `pwd`
+echo ----
+echo The contents of PBS_O_WORKDIR:
+ls -ltr
+echo
+echo ----
+echo
+echo creating a file in PBS_O_WORKDIR
+whoami > whoami-pbs-o-workdir
+
+cd $TMPDIR
+echo ----
+echo TMPDIR IS `pwd`
+echo ----
+echo wait for 42 seconds
+sleep 42
+echo ----
+echo creating a file in TMPDIR
+whoami > whoami-tmpdir
+
+# copy the file back to the output subdirectory
+pbsdcp -g $TMPDIR/whoami-tmpdir $PBS_O_WORKDIR/output
+
+echo ----
+echo Job ended at `date`

data/test/fixtures/app-template-rendered/params.yml

@@ -0,0 +1,8 @@
+---
+geometry: 'test.stl'
+walltime: '00:30:00'
+nodes: 'nodes=1:ppn=12'
+job_name: 'foobar'
+initial_temp: 80
+density: 0.0027
+coef: 1

data/test/fixtures/app-template-rendered/test/job.sh

@@ -0,0 +1,40 @@
+#PBS -l walltime=00:30:00
+#PBS -l nodes=1:ppn=12
+#PBS -N foobar
+#PBS -j oe
+#PBS -r n
+
+echo ----
+echo Job started at `date`
+echo ----
+echo This job is working on compute node `cat $PBS_NODEFILE`
+
+echo "TEMP IS 80"
+
+cd $PBS_O_WORKDIR
+echo show what PBS_O_WORKDIR is
+echo PBS_O_WORKDIR IS `pwd`
+echo ----
+echo The contents of PBS_O_WORKDIR:
+ls -ltr
+echo
+echo ----
+echo
+echo creating a file in PBS_O_WORKDIR
+whoami > whoami-pbs-o-workdir
+
+cd $TMPDIR
+echo ----
+echo TMPDIR IS `pwd`
+echo ----
+echo wait for 42 seconds
+sleep 42
+echo ----
+echo creating a file in TMPDIR
+whoami > whoami-tmpdir
+
+# copy the file back to the output subdirectory
+pbsdcp -g $TMPDIR/whoami-tmpdir $PBS_O_WORKDIR/output
+
+echo ----
+echo Job ended at `date`

data/test/fixtures/app-template/job.sh.mustache

@@ -0,0 +1,40 @@
+#PBS -l walltime={{walltime}}
+#PBS -l {{nodes}}
+#PBS -N {{job_name}}
+#PBS -j oe
+#PBS -r n
+
+echo ----
+echo Job started at `date`
+echo ----
+echo This job is working on compute node `cat $PBS_NODEFILE`
+
+echo "TEMP IS {{initial_temp}}"
+
+cd $PBS_O_WORKDIR
+echo show what PBS_O_WORKDIR is
+echo PBS_O_WORKDIR IS `pwd`
+echo ----
+echo The contents of PBS_O_WORKDIR:
+ls -ltr
+echo
+echo ----
+echo
+echo creating a file in PBS_O_WORKDIR
+whoami > whoami-pbs-o-workdir
+
+cd $TMPDIR
+echo ----
+echo TMPDIR IS `pwd`
+echo ----
+echo wait for 42 seconds
+sleep 42
+echo ----
+echo creating a file in TMPDIR
+whoami > whoami-tmpdir
+
+# copy the file back to the output subdirectory
+pbsdcp -g $TMPDIR/whoami-tmpdir $PBS_O_WORKDIR/output
+
+echo ----
+echo Job ended at `date`

data/test/fixtures/app-template/params.yml.mustache

@@ -0,0 +1,8 @@
+---
+geometry: '{{geometry}}'
+walltime: '{{walltime}}'
+nodes: '{{nodes}}'
+job_name: '{{job_name}}'
+initial_temp: {{initial_temp}}
+density: {{density}}
+coef: {{coef}}

data/test/fixtures/app-template/test/job.sh.mustache

@@ -0,0 +1,40 @@
+#PBS -l walltime={{walltime}}
+#PBS -l {{nodes}}
+#PBS -N {{job_name}}
+#PBS -j oe
+#PBS -r n
+
+echo ----
+echo Job started at `date`
+echo ----
+echo This job is working on compute node `cat $PBS_NODEFILE`
+
+echo "TEMP IS {{initial_temp}}"
+
+cd $PBS_O_WORKDIR
+echo show what PBS_O_WORKDIR is
+echo PBS_O_WORKDIR IS `pwd`
+echo ----
+echo The contents of PBS_O_WORKDIR:
+ls -ltr
+echo
+echo ----
+echo
+echo creating a file in PBS_O_WORKDIR
+whoami > whoami-pbs-o-workdir
+
+cd $TMPDIR
+echo ----
+echo TMPDIR IS `pwd`
+echo ----
+echo wait for 42 seconds
+sleep 42
+echo ----
+echo creating a file in TMPDIR
+whoami > whoami-tmpdir
+
+# copy the file back to the output subdirectory
+pbsdcp -g $TMPDIR/whoami-tmpdir $PBS_O_WORKDIR/output
+
+echo ----
+echo Job ended at `date`

data/test/fixtures/oakley.sh

@@ -0,0 +1,14 @@
+#PBS -l walltime=00:30:00
+#PBS -l nodes=1:ppn=12
+#PBS -S /bin/bash
+#PBS -q @oak-batch.osc.edu
+#PBS -N foobar
+#PBS -j oe
+#PBS -r n
+
+echo ----
+echo Job started at `date`
+echo ----
+echo This job is working on compute node `cat $PBS_NODEFILE`
+
+echo "TEMP IS 80"

data/test/fixtures/quick.sh

@@ -0,0 +1,14 @@
+#PBS -l walltime=00:30:00
+#PBS -l nodes=1:ppn=12
+#PBS -S /bin/bash
+#PBS -q @quick-batch.osc.edu
+#PBS -N foobar
+#PBS -j oe
+#PBS -r n
+
+echo ----
+echo Job started at `date`
+echo ----
+echo This job is working on compute node `cat $PBS_NODEFILE`
+
+echo "TEMP IS 80"

data/test/fixtures/ruby.sh

@@ -0,0 +1,14 @@
+#PBS -l walltime=00:30:00
+#PBS -l nodes=1:ppn=20
+#PBS -S /bin/bash
+#PBS -q @ruby-batch.osc.edu
+#PBS -N foobar
+#PBS -j oe
+#PBS -r n
+
+echo ----
+echo Job started at `date`
+echo ----
+echo This job is working on compute node `cat $PBS_NODEFILE`
+
+echo "TEMP IS 80"

data/test/test_job.rb

@@ -0,0 +1,179 @@
+require 'minitest/autorun'
+require 'osc/machete'
+
+class TestJob < Minitest::Test
+  def setup
+    @id1 = "16376371.opt-batch.osc.edu"
+    @id2 = "16376372.opt-batch.osc.edu"
+    
+    @jobdir = Pathname.new(Dir.mktmpdir)
+    @scriptname = "main.sh"
+    @scriptpath = @jobdir.join(@scriptname)
+    FileUtils.touch(@scriptpath)
+  end
+  
+  def teardown
+    @jobdir.rmtree if @jobdir.exist?
+  end
+  
+  def test_basic_job
+    job = OSC::Machete::Job.new(script: @scriptpath)
+    assert_equal job.path.to_s, @jobdir.to_s
+    assert_equal job.script_name, @scriptname
+  end
+  
+  # test outgoing qsub messages send correct dependency arguments
+  # when setting up a job that depends on another job
+  def test_job_dependency_afterany
+    # create first job and expect qsub to work as it does before
+    torque1 = OSC::Machete::TorqueHelper.new
+    torque1.expects(:qsub).with(@scriptname, has_entry(depends_on: {})).returns(@id1)
+    job1 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque1
+    
+    # create second job that depends on the first and expect qsub to send pbsid of the first job
+    # which will not be known until the first job qsub-ed and 16376371.opt-batch.osc.edu is returned
+    torque2 = OSC::Machete::TorqueHelper.new
+    torque2.expects(:qsub).with(@scriptname, has_entry(depends_on: { afterany: [@id1] })).returns(@id2)
+    job2 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque2
+    
+    # add the dependency and submit the job
+    job2.afterany job1
+    job2.submit
+    
+    # assertions
+    assert job1.submitted?, "dependent job not submitted"
+    assert job2.submitted?, "job not submitted"
+    
+    assert_equal @id1, job1.pbsid
+    assert_equal @id2, job2.pbsid
+  end
+  
+  # test outgoing qsub messages send correct dependency arguments
+  # when setting up a job that depends on another job
+  def test_job_dependency_afterok
+    # create first job and expect qsub to work as it does before
+    torque1 = OSC::Machete::TorqueHelper.new
+    torque1.expects(:qsub).with(@scriptname, has_entry(depends_on: {})).returns(@id1)
+    job1 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque1
+
+    # create second job that depends on the first and expect qsub to send pbsid of the first job
+    # which will not be known until the first job qsub-ed and 16376371.opt-batch.osc.edu is returned
+    torque2 = OSC::Machete::TorqueHelper.new
+    torque2.expects(:qsub).with(@scriptname, has_entry(depends_on: { afterok: [@id1] })).returns(@id2)
+    job2 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque2
+
+    # add the dependency and submit the job
+    job2.afterok job1
+    job2.submit
+
+    # assertions
+    assert job1.submitted?, "dependent job not submitted"
+    assert job2.submitted?, "job not submitted"
+
+    assert_equal @id1, job1.pbsid
+    assert_equal @id2, job2.pbsid
+  end
+  
+  # here we repeat the above test but when setting up the dependency we chain
+  # OSC::Machete::Job.new(...).afterok(...)
+  # as long as afterok returns self, chaining will work
+  def test_job_dependency_afterok_chaining
+    # create first job and expect qsub to work as it does before
+    torque1 = OSC::Machete::TorqueHelper.new
+    torque1.expects(:qsub).with(@scriptname, has_entry(depends_on: {})).returns(@id1)
+    job1 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque1
+
+    # create second job that depends on the first and expect qsub to send pbsid of the first job
+    # which will not be known until the first job qsub-ed and 16376371.opt-batch.osc.edu is returned
+    torque2 = OSC::Machete::TorqueHelper.new
+    torque2.expects(:qsub).with(@scriptname, has_entry(depends_on: { afterok: [@id1] })).returns(@id2)
+    job2 = OSC::Machete::Job.new(script: @scriptpath, torque_helper: torque2).afterok(job1)
+    job2.submit
+  end
+  
+  def test_job_dependency_after
+    # create first job and expect qsub to work as it does before
+    torque1 = OSC::Machete::TorqueHelper.new
+    torque1.expects(:qsub).with(@scriptname, has_entry(depends_on: {})).returns(@id1)
+    job1 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque1
+
+    # create second job that depends on the first and expect qsub to send pbsid of the first job
+    # which will not be known until the first job qsub-ed and 16376371.opt-batch.osc.edu is returned
+    torque2 = OSC::Machete::TorqueHelper.new
+    torque2.expects(:qsub).with(@scriptname, has_entry(depends_on: { after: [@id1] })).returns(@id2)
+    job2 = OSC::Machete::Job.new script: @scriptpath, torque_helper: torque2
+
+    # add the dependency and submit the job
+    job2.after job1
+    job2.submit
+  end
+
+  def test_job_status
+    torque1 = OSC::Machete::TorqueHelper.new
+    torque1.expects(:qstat).with(@id1, any_parameters).returns(OSC::Machete::Status.running)
+    job = OSC::Machete::Job.new pbsid: @id1, torque_helper: torque1
+
+    assert_equal job.status, OSC::Machete::Status.running
+
+    job = OSC::Machete::Job.new script: @scriptpath
+    assert_equal job.status, OSC::Machete::Status.not_submitted
+  end
+  
+  
+  def test_job_delete
+    # FIXME: the unit tests in this file are not dry...
+    # FIXME: rethink the interface: should delete return true if a job was actually deleted?
+    
+    torque1 = OSC::Machete::TorqueHelper.new
+    torque1.expects(:qdel).with(@id1, any_parameters).returns(true)
+    job = OSC::Machete::Job.new(script: @scriptpath, pbsid: @id1, torque_helper: torque1)
+    job.delete
+    assert @jobdir.exist?, "deleting job by default should not deleted the directory too"
+    
+    torque2 = OSC::Machete::TorqueHelper.new
+    torque2.expects(:qdel).with(@id1, any_parameters).returns(true)
+    job = OSC::Machete::Job.new(script: @scriptpath, pbsid: @id1, torque_helper: torque2)
+    job.delete(rmdir: true)
+    
+    assert ! @jobdir.exist?, "deleting job and specifying rmdir:true should have deleted the directory too"
+  end
+  
+  def test_job_dependency_delete
+    torque1 = OSC::Machete::TorqueHelper.new
+    torque1.expects(:qdel).with(@id1, any_parameters).returns(true)
+    torque1.expects(:qdel).with(@id2, any_parameters).returns(true)
+    job1 = OSC::Machete::Job.new(script: @scriptpath, pbsid: @id1, torque_helper: torque1)
+    job2 = OSC::Machete::Job.new(script: @scriptpath, pbsid: @id2, torque_helper: torque1)
+    
+    job2.afterok job1
+    
+    # we want to be able to safely delete two jobs that share the same pbs_work_dir and don't
+    # want to be able to call rmdir: true on both without worrying that the first one deleted
+    # the actual directory so the second might fail
+    # both should succeed (if the directory doesn't exist, just ignore that step)
+    job1.delete(rmdir: true)
+    job2.delete(rmdir: true)
+  end
+
+
+  def assert_submit_qsubs_with_account_string(args, account_string: nil)
+    # create first job and expect qsub to work as it does before
+    torque1 = OSC::Machete::TorqueHelper.new
+    torque1.expects(:qsub).with(@scriptname, has_entry(account_string: account_string)).returns(@id1)
+    OSC::Machete::Job.new(args.merge({script: @scriptpath, torque_helper: torque1})).submit
+  end
+
+  def test_account_string_not_set
+    # with default to nil we ensure that if we pass in account_string to
+    # initializer its used in qsub
+    assert_submit_qsubs_with_account_string({}, account_string: nil)
+    assert_submit_qsubs_with_account_string({account_string: "PZ3"}, account_string: "PZ3")
+
+    # if default set, we use default unless account_string is passed
+    OSC::Machete::Job.default_account_string = "FOO"
+    assert_submit_qsubs_with_account_string({account_string: "PZ3"}, account_string: "PZ3")
+    assert_submit_qsubs_with_account_string({}, account_string: "FOO")
+
+    OSC::Machete::Job.default_account_string = nil
+  end
+end