osc-machete 1.1.3

data/lib/osc/machete/job_dir.rb

@@ -0,0 +1,56 @@
+# helper class to create job directories
+class OSC::Machete::JobDir
+  def initialize(parent_directory)
+    @target = Pathname.new(parent_directory).cleanpath
+  end
+
+  # Returns a unique path for a job
+  #
+  # @return [String] A path of a unique job directory as string.
+  def new_jobdir
+    @target + unique_dir
+  end
+
+  #FIXME: BELOW METHODS SHOULD BE PRIVATE
+
+  # return true if the string is a job dir name
+  def jobdir_name?(name)
+    name[/^\d+$/]
+  end
+
+  # return true if Pathname is a job directory
+  # FIXME: this is not used anywhere; remove it?
+  def jobdir?(path)
+    jobdir_name?(path.basename.to_s) && path.directory?
+  end
+
+  # get a list of all job directories
+  # FIXME: this is not used anywhere; remove it?
+  def jobdirs
+    @target.exist? ? @target.children.select { |i| jobdir?(i) } : []
+  end
+
+
+  # get a list of directories in the target directory
+  # FIXME: this is not used anywhere; remove it?
+  def targetdirs
+    @target.exist? ? @target.children.select(&:directory?) : []
+  end
+
+  # find the next unique integer name for a job directory
+  def unique_dir
+    taken_ints = taken_paths.map { |path| path.basename.to_s.to_i }
+    (taken_ints.count > 0) ? (taken_ints.max + 1).to_s : 1.to_s
+  end
+
+  private
+
+  # paths that are unavailable for creating a new job directory
+  def taken_paths
+    if @target.exist?
+      @target.children.select { |path| jobdir_name?(path.basename.to_s) }
+    else
+      []
+    end
+  end
+end

data/lib/osc/machete/location.rb

@@ -0,0 +1,91 @@
+require 'pathname'
+require 'mustache'
+
+# A util class with methods used with staging a simulation template directory.
+# Use it by wrapping a file path (either string or Pathname object).
+# For example, if I have a template directory at "/nfs/05/efranz/template"
+# I can recursivly copy the template directory:
+#
+#     target = "/nfs/05/efranz/simulations/1"
+#     simulation = Location.new("/nfs/05/efranz/template").copy_to(target)
+#
+# Then I can recursively render all the mustache templates in the copied directory,
+# renaming each file from XXXX.mustache to XXXX:
+#
+#     simulation.render(iterations: 20, geometry: "/nfs/05/efranz/geos/fan.stl")
+#
+class OSC::Machete::Location
+  # URIs, Paths, rendering, and copying
+  # this should be refactored into separate objects
+
+  # @param path  either string, Pathname, or Machete::Location object
+  def initialize(path)
+    @path = Pathname.new(path.to_s).cleanpath
+    @template_ext = ".mustache"
+  end
+
+  # @return [String] The location path as String.
+  def to_s
+    @path.to_s
+  end
+
+  # Copies the data in a Location to a destination path using rsync.
+  #
+  # @param [String, Pathname] dest The target location path.
+  # @return [Location] The target location path wrapped by Location instance.
+  def copy_to(dest)
+    # @path has / auto-dropped, so we add it to make sure we copy everything
+    # in the old directory to the new
+    destloc = self.class.new(dest)
+    `rsync -r --exclude='.svn' --exclude='.git' --exclude='.gitignore' --filter=':- .gitignore' #{@path.to_s}/ #{destloc.to_s}`
+
+    # return target location so we can chain method
+    destloc
+  end
+
+  # **This should be a private method**
+  #
+  # Get a list of template files in this Location, where a template file is a
+  # file with the extension .mustache
+  #
+  # @return [Array<String>] list of template files in directory (recursively searched)
+  def template_files
+    if @path.directory?
+      Dir.glob(File.join(@path, "**/*#{@template_ext}"))
+    else
+      @path.to_s.end_with? @template_ext ? [@path.to_s] : []
+    end
+  end
+
+  # Render each mustache template and rename the file, removing the extension
+  # that indicates it is a template file i.e. `.mustache`.
+  #
+  # @param [Hash] params the "context" or "hash" for use when rendering mustache templates
+  # @param [Hash] options to modify rendering behavior
+  # @option options [Boolean] :replace (true) if true will delete the template file after the rendered file is created
+  # @return [self] returns self for optional chaining
+  def render(params, options = {})
+    # custom_delimiters = options['delimeters'] || nil
+    replace_template_files = options[:replace].nil? ? true : options[:replace]
+
+    renderer = Mustache.new
+
+    template_files.each do |template|
+      rendered_file = template.chomp(@template_ext)
+
+      rendered_string = nil
+      File.open(template, 'r') do |f|
+        rendered_string = renderer.render(f.read, params)
+      end
+      # boo...
+      # rendered_string = renderer.render_file(template, params)
+
+      File.open(rendered_file, 'w') { |f| f.write(rendered_string) }
+
+      FileUtils.rm template if replace_template_files
+    end
+
+    # return self so this can be at the end of a chained method
+    self
+  end
+end

data/lib/osc/machete/process.rb

@@ -0,0 +1,32 @@
+# Class that maintains the User and additional methods for the process.
+# Helper methods provided use the Process module underneath.
+#
+class OSC::Machete::Process
+
+  def initialize
+    @user = OSC::Machete::User.from_uid(Process.uid)
+  end
+
+  # The system name of the process user
+  def username
+    @user.name
+  end
+
+  # use gid not egid
+  def groupname
+    Etc.getgrgid(Process.gid).name
+  end
+
+  # has the group membership changed since this process started?
+  def group_membership_changed?
+    Process.groups.uniq.sort != @user.groups
+  end
+
+  # The home directory path of the process user.
+  #
+  # @return [String] The directory path.
+  def home
+    @user.home
+  end
+
+end

data/lib/osc/machete/status.rb

@@ -0,0 +1,190 @@
+# Value object representing job status independent of underlying resource manager.
+#
+# All of the possible Torque statuses are not represented here. Its the
+# responsibility of the Torque adapter (TorqueHelper) to create the appropriate
+# Status object to represent the Torque status.
+#
+# The possible values are: Undetermined, Not Submitted, Passed, Failed, Held, Queued, Running, Suspended
+#
+class OSC::Machete::Status
+  include Comparable
+  
+  attr_reader :char
+
+  # C Job is passed (completed successfully)
+  # F Job is failed (completed with errors)
+  # H Job is held.
+  # Q Job is queued, eligible to run or routed.
+  # R Job is running.
+  #
+  # U Status is unavailable (null status object)
+  VALUES = [["U", "undetermined"], [nil, "not_submitted"], ["C", "passed"], ["F", "failed"],
+            ["H", "held"], ["Q", "queued"], ["R", "running"], ["S", "suspended"]]
+  private_constant :VALUES
+
+  # A hashed version of the values array.
+  VALUES_HASH = Hash[VALUES]
+  private_constant :VALUES_HASH
+
+  # An array of status char values by precedence.
+  #
+  # @example
+  #   OSC::Machete::Status::PRECEDENCE #=> ["U", nil, "C", "F", "H", "Q", "R", "S"]
+  PRECEDENCE = VALUES.map(&:first)
+  private_constant :PRECEDENCE
+
+  # Get an array of all the possible Status values
+  #
+  # @return [Array<Status>] - all possible Status values
+  def self.values
+    VALUES.map{ |v| OSC::Machete::Status.new(v.first) }
+  end
+
+  # Get an array of all the possible active Status values
+  #
+  # @return [Array<Status>] - all possible active Status values
+  def self.active_values
+    values.select(&:active?)
+  end
+
+  # Get an array of all the possible completed Status values
+  #
+  # @return [Array<Status>] - all possible completed Status values
+  def self.completed_values
+    values.select(&:completed?)
+  end
+
+
+  # TODO: these methods are previously declared so we can document them easily
+  # if there is a better way to document class methods we'll do that
+
+  # @return [Status]
+  def self.undetermined() end
+  # @return [Status] 
+  def self.not_submitted() end
+  # @return [Status]
+  def self.passed() end
+  # @return [Status] 
+  def self.failed() end
+  # @return [Status]
+  def self.running() end
+  # @return [Status] 
+  def self.queued() end
+  # @return [Status] 
+  def self.held() end
+  # @return [Status] 
+  def self.suspended() end
+
+  class << self
+    VALUES_HASH.each do |char, name|
+      define_method(name) do
+        OSC::Machete::Status.new(char)
+      end
+    end
+  end
+
+  # @!method undetermined?
+  #   the Status value Null object
+  #   @return [Boolean] true if undetermined
+  # @!method not_submitted?
+  #   @return [Boolean] true if not_submitted
+  # @!method failed?
+  #   @return [Boolean] true if failed
+  # @!method passed?
+  #   @return [Boolean] true if passed
+  # @!method held?
+  #   @return [Boolean] true if held
+  # @!method queued?
+  #   @return [Boolean] true if queued
+  # @!method running?
+  #   @return [Boolean] true if running
+  # @!method suspended?
+  #   @return [Boolean] true if suspended
+  VALUES_HASH.each do |char, name|
+    define_method("#{name}?") do
+      self == OSC::Machete::Status.new(char)
+    end
+  end
+
+  def initialize(char)
+    # char could be a status object or a string
+    @char = (char.respond_to?(:char) ? char.char : char).to_s.upcase
+    @char = nil if @char.empty?
+
+    # if invalid status value char, default to undetermined
+    @char = self.class.undetermined.char unless VALUES_HASH.has_key?(@char)
+  end
+
+  # @return [Boolean] true if submitted
+  def submitted?
+    ! (not_submitted? || undetermined?)
+  end
+
+  # @return [Boolean] true if in active state (running, queued, held, suspended)
+  def active?
+    running? || queued? || held? || suspended?
+  end
+
+  # @return [Boolean] true if in completed state (passed or failed)
+  def completed?
+    passed? || failed?
+  end
+
+  # Return a readable string of the status
+  #
+  # @example Running
+  #     OSC::Machete::Status.running.to_s #=> "Running"
+  #
+  # @return [String] The status value as a formatted string
+  def to_s
+    # FIXME: ActiveSupport  replace with .humanize and simpler datastructure
+     VALUES_HASH[@char].split("_").map(&:capitalize).join(" ")
+  end
+
+  # Return the a StatusValue object based on the highest precedence of the two objects.
+  #
+  # @example One job is running and a dependent job is queued.
+  #   OSC::Machete::Status.running + OSC::Machete::Status.queued #=> Running
+  #
+  # Return [OSC::Machete::Status] The max status by precedence
+  def +(other)
+    [self, other].max
+  end
+
+  # The comparison operator for sorting values.
+  #
+  # @return [Integer] Comparison value based on precedence
+  def <=>(other)
+    precedence <=> other.precedence
+  end
+
+  # Boolean evaluation of Status object equality.
+  #
+  # @return [Boolean] True if the values are the same
+  def eql?(other)
+    # compare Status to Status OR "C" to Status
+    (other.respond_to?(:char) ? other.char : other) == char
+  end
+
+  # Boolean evaluation of Status object equality.
+  #
+  # @return [Boolean] True if the values are the same
+  def ==(other)
+    self.eql?(other)
+  end
+
+  # Return a hash based on the char value of the object.
+  #
+  # @return [Fixnum] A hash value of the status char
+  def hash
+    @char.hash
+  end
+
+  # Return the ordinal position of the status in the precidence list
+  #
+  # @return [Integer] The order of precedence for the object
+  def precedence
+    # Hashes enumerate their values in the order that the corresponding keys were inserted
+    PRECEDENCE.index(@char)
+  end
+end

data/lib/osc/machete/torque_helper.rb

@@ -0,0 +1,190 @@
+require 'pbs'
+
+# == Helper object: ruby interface to torque shell commands
+# in the same vein as stdlib's Shell which
+# "implements an idiomatic Ruby interface for common UNIX shell commands"
+# also helps to have these separate so we can use a mock shell for unit tests
+#
+# == FIXME: This contains no state whatsoever. It should probably be changed into a module.
+class OSC::Machete::TorqueHelper
+
+  # Alias to initialize a new object.
+  def self.default
+    self::new()
+  end
+
+  # Returns an OSC::Machete::Status ValueObject for a char
+  #
+  # @param [String] char The Torque status char
+  #
+  # @example Completed
+  #   status_for_char("C") #=> OSC::Machete::Status.completed
+  # @example Queued
+  #   status_for_char("W") #=> OSC::Machete::Status.queued
+  #
+  # @return [OSC::Machete::Status] The status corresponding to the char
+  def status_for_char(char)
+    case char
+    when "C", nil
+      OSC::Machete::Status.passed
+    when "Q", "T", "W" # T W happen before job starts
+      OSC::Machete::Status.queued
+    when "H"
+      OSC::Machete::Status.held
+    else
+      # all other statuses considerd "running" state
+      # including S, E, etc.
+      # see http://docs.adaptivecomputing.com/torque/4-1-3/Content/topics/commands/qstat.htm
+      OSC::Machete::Status.running
+    end
+  end
+
+  #*TODO:*
+  # consider using cocaine gem
+  # consider using Shellwords and other tools
+
+  # usage: <tt>qsub("/path/to/script")</tt> or
+  #        <tt>qsub("/path/to/script", depends_on: { afterany: ["1234.oak-batch.osc.edu"] })</tt>
+  #
+  # Where depends_on is a hash with key being dependency type and array containing the
+  # arguments. See documentation on dependency_list in qsub man pages for details.
+  #
+  # Bills against the project specified by the primary group of the user.
+  def qsub(script, host: nil, depends_on: {}, account_string: nil)
+    # if the script is set to run on Oakley in PBS headers
+    # this is to obviate current torque filter defect in which
+    # a script with PBS header set to specify oak-batch ends
+    # isn't properly handled and the job gets limited to 4GB
+    pbs_job = get_pbs_job( host.nil? ? get_pbs_conn(script: script) : get_pbs_conn(host: host) )
+
+    headers = { depend: qsub_dependencies_header(depends_on) }
+    headers.clear if headers[:depend].empty?
+
+    # currently we set the billable project to the name of the primary group
+    # this will probably be both SUPERCOMPUTER CENTER SPECIFIC and must change
+    # when we want to enable our users at OSC to specify which billable project
+    # to bill against
+    if account_string
+      headers[PBS::ATTR[:A]] = account_string
+    elsif account_string_valid_project?(default_account_string)
+      headers[PBS::ATTR[:A]] = default_account_string
+    end
+
+    pbs_job.submit(file: script, headers: headers, qsub: true).id
+  end
+
+  # convert dependencies hash to a PBS header string
+  def qsub_dependencies_header(depends_on = {})
+    depends_on.map { |x|
+      x.first.to_s + ":" + Array(x.last).join(":") unless Array(x.last).empty?
+    }.compact.join(",")
+  end
+
+  # return the account string required for accounting purposes
+  # having this in a separate method is useful for monkeypatching in short term
+  # or overridding with a subclass you pass into OSC::Machete::Job
+  #
+  # FIXME: this may belong on OSC::Machete::User; but it is OSC specific...
+  #
+  # @return [String] the project name that job submission should be billed against
+  def default_account_string
+    OSC::Machete::Process.new.groupname
+  end
+
+  def account_string_valid_project?(account_string)
+    /^P./ =~ account_string
+  end
+
+  # Performs a qstat request on a single job.
+  #
+  # **FIXME: this might not belong here!**
+  #
+  # @param [String] pbsid The pbsid of the job to inspect.
+  #
+  # @return [Status] The job state
+  def qstat(pbsid, host: nil)
+
+    # Create a PBS::Job object based on the pbsid or the optional host param
+    pbs_conn = host.nil? ? get_pbs_conn(pbsid: pbsid.to_s) : get_pbs_conn(host: host)
+    pbs_job = get_pbs_job(pbs_conn, pbsid)
+
+    job_status = pbs_job.status
+    # Get the status char value from the job.
+    status_for_char job_status[:attribs][:job_state][0]
+
+  rescue PBS::UnkjobidError => err
+    OSC::Machete::Status.passed
+  end
+
+  # Perform a qdel command on a single job.
+  #
+  # @param [String] pbsid The pbsid of the job to be deleted.
+  #
+  # @return [nil]
+  def qdel(pbsid, host: nil)
+
+    pbs_conn   =   host.nil? ? get_pbs_conn(pbsid: pbsid.to_s) : get_pbs_conn(host: host)
+    pbs_job    =   get_pbs_job(pbs_conn, pbsid.to_s)
+
+    pbs_job.delete
+
+  rescue PBS::UnkjobidError => err
+    # Common use case where trying to delete a job that is no longer in the system.
+  end
+
+  private
+
+    # Factory to return a PBS::Job object
+    def get_pbs_job(conn, pbsid=nil)
+      pbsid.nil? ? PBS::Job.new(conn: conn) : PBS::Job.new(conn: conn, id: pbsid.to_s)
+    end
+
+    # Returns a PBS connection object
+    #
+    # @option [:script] A PBS script with headers as string
+    # @option [:pbsid] A valid pbsid as string
+    #
+    # @return [PBS::Conn] A connection option for the PBS host (Default: Oakley)
+    def get_pbs_conn(options={})
+      if options[:script]
+        PBS::Conn.batch(host_from_script_pbs_header(options[:script]))
+      elsif options[:pbsid]
+        PBS::Conn.batch(host_from_pbsid(options[:pbsid]))
+      elsif options[:host]
+        PBS::Conn.batch(options[:host])
+      else
+        PBS::Conn.batch("oakley")
+      end
+    end
+
+    # return the name of the host to use based on the pbs header
+    # TODO: Think of a more efficient way to do this.
+    def host_from_script_pbs_header(script)
+      if (File.open(script) { |f| f.read =~ /#PBS -q @oak-batch/ })
+        "oakley"
+      elsif (File.open(script) { |f| f.read =~ /#PBS -q @opt-batch/ })
+        "glenn"
+      elsif (File.open(script) { |f| f.read =~ /#PBS -q @ruby-batch/ })
+        "ruby"
+      elsif (File.open(script) { |f| f.read =~ /#PBS -q @quick-batch/ })
+        "quick"
+      else
+        "oakley"  # DEFAULT
+      end
+    end
+
+    # Return the PBS host string based on a full pbsid string
+    def host_from_pbsid(pbsid)
+      if (pbsid =~ /oak-batch/ )
+        "oakley"
+      elsif (pbsid =~ /opt-batch/ )
+        "glenn"
+      elsif (pbsid.to_s =~ /^\d+$/ )
+        "ruby"
+      elsif (pbsid =~ /quick/ )
+        "quick"
+      else
+        "oakley"  # DEFAULT
+      end
+    end
+end