pbs 1.1.4 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3af819b077ee91da66818d771629351f3a82f157
-  data.tar.gz: c732e8ea53bd0640b941421644e0555a1dcd2180
+  metadata.gz: 6b983a1e056e48ac1fb0973f565910ecaa7b59e8
+  data.tar.gz: 72ea5ac4e9b96137f9d234f44ea9912bb4aef298
 SHA512:
-  metadata.gz: 45f4ba1bbc056a036a22276669d9a5db484b9b5308578ca4b4bdd90af1080ed9e1309cbaacc7b41a20e8e8c0b830244bbaedaa067f8fc9f38da832f4446c1d3a
-  data.tar.gz: b88452f9007a1c6972808764f074adddfea6da6ca36d04f615228bfcffc680937cc3c601d618c8f29fc8f4590f8976d07d052229294704462c504c2660d9ec5d
+  metadata.gz: afe01c15267f13df46e452dab396cf568af6751fe4a92648f174b2d3a2af30443b902115938a28c9e4e7e37b0e436e1d37f13f616c3eff0a61c68f1912956c64
+  data.tar.gz: ef15244b9433ffff530d12da0a90a01a3e7d017c127028005f3831322f7cfac1a7c84c07cdf0c9ffdfc64d8ba3c47f53ae62cec06bd1d37d7141cfdf287e0a61

data/CHANGELOG.md

@@ -0,0 +1,8 @@
+## Unreleased
+
+## 2.0.0 (2016-08-05)
+
+Features:
+
+  - initial release of v2.0.0
+  - added a changelog

data/README.md

@@ -2,33 +2,217 @@
 
 ## Description
 
-Trimmed down Ruby wrapper for the Torque C Library utilizing Ruby-FFI.
-
-## Requirements
-
-At minimum you will need:
-* Ruby 2.0
-* Ruby-FFI gem
-* Torque >= 4.2.10
+Ruby wrapper for the Torque C Library utilizing Ruby-FFI. This has been
+successfully tested with Torque 4.2.10 and greater. Your mileage may vary.
 
 ## Installation
 
 Add this to your application's Gemfile:
 
 ```ruby
-  gem 'pbs'
+gem 'pbs'
 ```
 
 And then execute:
 
 ```bash
-  $ bundle install
+$ bundle install
 ```
 
 ## Usage
 
-Most useful features are outlined in the `examples/simplejob.rb` provided. To run this simple example, type:
+All communication with a specific batch server is handled through the `Batch`
+object. You can generate this object for a given batch server and Torque client
+installation as:
 
-```bash
-  $ ruby -Ilib examples/simplejob.rb
+```ruby
+# Create new batch object for OSC's Oakley batch server
+oakley = PBS::Batch.new(host: 'oak-batch.osc.edu', prefix: '/usr/local/torque/default')
+
+# Get status information for this batch server
+# see http://linux.die.net/man/7/pbs_server_attributes
+oakley.get_status
+#=>
+#{
+#  "oak-batch.osc.edu:15001" => {
+#    :server_state  => "Idle",
+#    :total_jobs    => "2514",
+#    :default_queue => "batch",
+#    ...
+#  }
+#}
+
+# Get status information but only filter through specific attributes
+oakley.get_status(filters: [:server_state, :total_jobs])
+#=>
+#{
+#  "oak-batch.osc.edu:15001" => {
+#    :server_state  => "Idle",
+#    :total_jobs    => "2514"
+#  }
+#}
+```
+
+You can also query information about the nodes, queues, and jobs running on
+this batch server:
+
+```ruby
+# Get list of nodes from batch server
+b.get_nodes
+#=>
+#{
+#  "n0003" => {
+#    :state       => "free",
+#    :power_state => "Running",
+#    :np          => "12",
+#    ...
+#  },
+#  "n0004" => {
+#    :state       => "free",
+#    :power_state => "Running",
+#    :np          => "12",
+#    ...
+#  }, ...
+#}
+
+# To get info about a single node
+b.get_node("n0003")
+#=> { ... }
+
+# Get list of queues from batch server
+# see http://linux.die.net/man/7/pbs_queue_attributes
+b.get_queues
+#=>
+#{
+#  "batch" => {
+#    :queue_type => "Route",
+#    :total_jobs => "2",
+#    :enabled    => "True",
+#    ...
+#  },
+#  "serial" => {
+#    :queue_type => "Execution",
+#    :total_jobs => "2386",
+#    :enabled    => "True",
+#    ...
+#  }, ...
+#}
+
+# To get info about a single queue
+b.get_queue("serial")
+#=> { ... }
+
+# Get list of jobs from batch server
+# see http://linux.die.net/man/7/pbs_server_attributes
+b.get_jobs
+#=>
+#{
+#  "6621251.oak-batch.osc.edu" => {
+#    :Job_Name  => "FEA_solver",
+#    :Job_Owner => "bob@oakley01.osc.edu",
+#    :job_state => "Q",
+#    ...
+#  },
+#  "6621252.oak-batch.osc.edu" => {
+#    :Job_Name  => "CFD_solver",
+#    :Job_Owner => "sally@oakley02.osc.edu",
+#    :job_state => "R",
+#    ...
+#  }, ...
+#}
+
+# To get info about a single job
+b.get_job("6621251.oak-batch.osc.edu")
+#=> { ... }
+```
+
+### Simple Job Submission
+
+To submit a script to the batch server:
+
+```ruby
+# Simple job submission
+job_id = b.submit_script("/path/to/script")
+#=> "7166037.oak-batch.osc.edu"
+
+# Get job information for this job
+b.get_job(job_id)
+#=> { ... }
+
+# Hold this job
+b.hold_job(job_id)
+
+# Release this job
+b.release_job(job_id)
+
+# Delete this job
+b.delete_job(job_id)
+```
+
+To submit a string to the batch server:
+
+```ruby
+# Submit a string to the batch server
+job_id = b.submit_string("sleep 60")
+```
+
+The above command will actually generate a temporary file on the local disk and
+submit that to the batch server before it is cleaned up.
+
+### Advanced Job Submission
+
+You can programmatically define the PBS directives of your choosing. They will
+override any set within the batch script.
+
+Define headers:
+
+```ruby
+# Define headers:
+#   -N job_name
+#   -j oe
+#   -o /path/to/output
+headers = {
+  PBS::ATTR[:N] => "job_name",
+  PBS::ATTR[:j] => "oe",
+  PBS::ATTR[:o] => "/path/to/output"
+}
+
+# or you can directly call the key
+headers = {
+  Job_Name: "job_name",
+  Join_Path: "oe",
+  Output_Path: "/path/to/output"
+}
+```
+
+Define resources (directives that begin with `-l`):
+
+```ruby
+# Define resources:
+#   -l nodes=1:ppn=12
+#   -l walltime=05:00:00
+resources = {
+  nodes: "1:ppn=12",
+  walltime: "05:00:00"
+}
+```
+
+Define environment variables (directive that begins with `-v`):
+
+```ruby
+# Define environment variables that will be exposed to batch job
+envvars = {
+  TOKEN: 'a8dsjf873js0k',
+  USE_GUI: 1
+}
+```
+
+Submit job with these directives:
+
+```ruby
+# Advanced job submission
+job_id = b.submit_script("/path/to/script", headers: headers, resources: resources, envvars: envvars)
+
+# Get job info
+b.get_job(job_id)
 ```

data/lib/pbs.rb

@@ -1,38 +1,9 @@
-require 'yaml'
-require 'socket'
-
 require_relative 'pbs/error'
 require_relative 'pbs/attributes'
 require_relative 'pbs/torque'
-require_relative 'pbs/conn'
-require_relative 'pbs/query'
-require_relative 'pbs/job'
+require_relative 'pbs/batch'
 require_relative 'pbs/version'
 
+# The main namespace for pbs
 module PBS
-    # Path to the batch config yaml file describing the batch servers for
-    # local batch schedulers.
-    # @return [String] Path to the batch config yaml file.
-    def self.default_batch_config_path
-      default_config = File.expand_path("../../config/batch.yml", __FILE__)
-      host_config = File.expand_path("../../config/#{Socket.gethostname}.yml", __FILE__)
-      File.file?(host_config) ? host_config : default_config
-    end
-
-    # Get the path to the batch config yaml file.
-    def self.batch_config_path
-      @batch_config_path ||= self.default_batch_config_path
-    end
-
-    # Set the path to the batch config yaml file.
-    # @param path [String] The path to the batch config yaml file.
-    def self.batch_config_path=(path)
-      @batch_config_path = File.expand_path(path)
-    end
-
-    # Hash generated from reading the batch config yaml file.
-    # @return [Hash] Batch configuration generated from config yaml file.
-    def self.batch_config
-      YAML.load_file(batch_config_path)
-    end
 end

data/lib/pbs/attributes.rb

@@ -1,12 +1,11 @@
-# Maintains a constant Hash of defined PBS attribute types
-#
-#   Includes:
-#     Attribute names used by user commands
-#     Additional job and general attribute names
-#     Additional queue attribute names
-#     Additional server attribute names
-#     Additional node attribute names
 module PBS
+  # Maintains a constant Hash of defined PBS attribute types
+  # Includes:
+  #   Attribute names used by user commands
+  #   Additional job and general attribute names
+  #   Additional queue attribute names
+  #   Additional server attribute names
+  #   Additional node attribute names
   ATTR = {
     # Attribute names used by user commands
     a:                 :Execution_Time,

data/lib/pbs/batch.rb

@@ -0,0 +1,334 @@
+require 'open3'
+
+module PBS
+  # Object used for simplified communication with a batch server
+  class Batch
+    # The host of the Torque batch server
+    # @example OSC's Oakley batch server
+    #   my_conn.host #=> "oak-batch.osc.edu"
+    # @return [String] the batch server host
+    attr_reader :host
+
+    # The path to the Torque client installation
+    # @example For Torque 5.0.0
+    #   my_conn.prefix.to_s #=> "/usr/local/torque/5.0.0"
+    # @return [Pathname, nil] path to torque installation
+    attr_reader :prefix
+
+    # @param host [#to_s] the batch server host
+    # @param prefix [#to_s, nil] path to torque installation
+    def initialize(host:, prefix: nil, **_)
+      @host    = host.to_s
+      @prefix  = Pathname.new(prefix) if prefix
+    end
+
+    # Convert object to hash
+    # @return [Hash] the hash describing this object
+    def to_h
+      {host: host, prefix: prefix}
+    end
+
+    # The comparison operator
+    # @param other [#to_h] batch server to compare against
+    # @return [Boolean] how batch servers compare
+    def ==(other)
+      to_h == other.to_h
+    end
+
+    # Checks whether two batch server objects are completely identical to each
+    # other
+    # @param other [Batch] batch server to compare against
+    # @return [Boolean] whether same objects
+    def eql?(other)
+      self.class == other.class && self == other
+    end
+
+    # Generates a hash value for this object
+    # @return [Fixnum] hash value of object
+    def hash
+      [self.class, to_h].hash
+    end
+
+    # Creates a connection to batch server and calls block in context of this
+    # connection
+    # @yieldparam cid [Fixnum] connection id from established batch server connection
+    # @yieldreturn the final value of the block
+    def connect(&block)
+      Torque.lib = prefix ? prefix.join('lib', 'libtorque.so') : nil
+      cid = Torque.pbs_connect(host)
+      Torque.raise_error(cid.abs) if cid < 0  # raise error if negative connection id
+      begin
+        value = yield cid
+      ensure
+        Torque.pbs_disconnect(cid)            # always close connection
+      end
+      Torque.check_for_error                  # check for errors at end
+      value
+    end
+
+    # Get a hash with status info for this batch server
+    # @example Status info for OSC Oakley batch server
+    #   my_conn.get_status
+    #   #=>
+    #   #{
+    #   #  "oak-batch.osc.edu:15001" => {
+    #   #    :server_state => "Idle",
+    #   #    ...
+    #   #  }
+    #   #}
+    # @param filters [Array<Symbol>] list of attribs to filter on
+    # @return [Hash] status info for batch server
+    def get_status(filters: [])
+      connect do |cid|
+        filters = PBS::Torque::Attrl.from_list filters
+        batch_status = Torque.pbs_statserver cid, filters, nil
+        batch_status.to_h.tap { Torque.pbs_statfree batch_status }
+      end
+    end
+
+    # Get a list of hashes of the queues on the batch server
+    # @example Status info for OSC Oakley queues
+    #   my_conn.get_queues
+    #   #=>
+    #   #{
+    #   #  "parallel" => {
+    #   #    :queue_type => "Execution",
+    #   #    ...
+    #   #  },
+    #   #  "serial" => {
+    #   #    :queue_type => "Execution",
+    #   #    ...
+    #   #  },
+    #   #  ...
+    #   #}
+    # @param id [#to_s] the id of requested information
+    # @param filters [Array<Symbol>] list of attribs to filter on
+    # @return [Hash] hash of details for the queues
+    def get_queues(id: '', filters: [])
+      connect do |cid|
+        filters = PBS::Torque::Attrl.from_list(filters)
+        batch_status = Torque.pbs_statque cid, id.to_s, filters, nil
+        batch_status.to_h.tap { Torque.pbs_statfree batch_status }
+      end
+    end
+
+    # Get info for given batch server's queue
+    # @example Status info for OSC Oakley's parallel queue
+    #   my_conn.get_queue("parallel")
+    #   #=>
+    #   #{
+    #   #  "parallel" => {
+    #   #    :queue_type => "Execution",
+    #   #    ...
+    #   #  }
+    #   #}
+    # @param (see @get_queues)
+    # @return [Hash] status info for the queue
+    def get_queue(id, **kwargs)
+      get_queues(id: id, **kwargs)
+    end
+
+
+    # Get a list of hashes of the nodes on the batch server
+    # @example Status info for OSC Oakley nodes
+    #   my_conn.get_nodes
+    #   #=>
+    #   #{
+    #   #  "n0001" => {
+    #   #    :np => "12",
+    #   #    ...
+    #   #  },
+    #   #  "n0002" => {
+    #   #    :np => "12",
+    #   #    ...
+    #   #  },
+    #   #  ...
+    #   #}
+    # @param id [#to_s] the id of requested information
+    # @param filters [Array<Symbol>] list of attribs to filter on
+    # @return [Hash] hash of details for nodes
+    def get_nodes(id: '', filters: [])
+      connect do |cid|
+        filters = PBS::Torque::Attrl.from_list(filters)
+        batch_status = Torque.pbs_statnode cid, id.to_s, filters, nil
+        batch_status.to_h.tap { Torque.pbs_statfree batch_status }
+      end
+    end
+
+    # Get info for given batch server's node
+    # @example Status info for OSC Oakley's 'n0001' node
+    #   my_conn.get_node('n0001')
+    #   #=>
+    #   #{
+    #   #  "n0001" => {
+    #   #    :np => "12",
+    #   #    ...
+    #   #  }
+    #   #}
+    # @param (see #get_nodes)
+    # @return [Hash] status info for the node
+    def get_node(id, **kwargs)
+      get_nodes(id: id, **kwargs)
+    end
+
+    # Get a list of hashes of the jobs on the batch server
+    # @example Status info for OSC Oakley jobs
+    #   my_conn.get_jobs
+    #   #=>
+    #   #{
+    #   #  "10219837.oak-batch.osc.edu" => {
+    #   #    :Job_Owner => "bob@oakley02.osc.edu",
+    #   #    :Job_Name => "CFD_Solver",
+    #   #    ...
+    #   #  },
+    #   #  "10219838.oak-batch.osc.edu" => {
+    #   #    :Job_Owner => "sally@oakley01.osc.edu",
+    #   #    :Job_Name => "FEA_Solver",
+    #   #    ...
+    #   #  },
+    #   #  ...
+    #   #}
+    # @param id [#to_s] the id of requested information
+    # @param filters [Array<Symbol>] list of attribs to filter on
+    # @return [Hash] hash of details for jobs
+    def get_jobs(id: '', filters: [])
+      connect do |cid|
+        filters = PBS::Torque::Attrl.from_list(filters)
+        batch_status = Torque.pbs_statjob cid, id.to_s, filters, nil
+        batch_status.to_h.tap { Torque.pbs_statfree batch_status }
+      end
+    end
+
+    # Get info for given batch server's job
+    # @example Status info for OSC Oakley's '10219837.oak-batch.osc.edu' job
+    #   my_conn.get_job('102719837.oak-batch.osc.edu')
+    #   #=>
+    #   #{
+    #   #  "10219837.oak-batch.osc.edu" => {
+    #   #    :Job_Owner => "bob@oakley02.osc.edu",
+    #   #    :Job_Name => "CFD_Solver",
+    #   #    ...
+    #   #  }
+    #   #}
+    # @param (see #get_jobs)
+    # @return [Hash] hash with details of job
+    def get_job(id, **kwargs)
+      get_jobs(id: id, **kwargs)
+    end
+
+    # Put specified job on hold
+    # Possible hold types:
+    #   :u => Available to the owner of the job, the batch operator and the batch administrator
+    #   :o => Available to the batch operator and the batch administrator
+    #   :s => Available to the batch administrator
+    # @example Put job '10219837.oak-batch.osc.edu' on hold
+    #   my_conn.hold_job('10219837.oak-batch.osc.edu')
+    # @param id [#to_s] the id of the job
+    # @param type [Symbol] type of hold to be applied
+    # @return [void]
+    def hold_job(id, type: :u)
+      connect do |cid|
+        Torque.pbs_holdjob cid, id.to_s, type.to_s, nil
+      end
+    end
+
+    # Release a specified job that is on hold
+    # Possible hold types:
+    #   :u => Available to the owner of the job, the batch operator and the batch administrator
+    #   :o => Available to the batch operator and the batch administrator
+    #   :s => Available to the batch administrator
+    # @example Release job '10219837.oak-batch.osc.edu' from hold
+    #   my_conn.release_job('10219837.oak-batch.osc.edu')
+    # @param id [#to_s] the id of the job
+    # @param type [Symbol] type of hold to be removed
+    # @return [void]
+    def release_job(id, type: :u)
+      connect do |cid|
+        Torque.pbs_rlsjob cid, id.to_s, type.to_s, nil
+      end
+    end
+
+    # Delete a specified job from batch server
+    # @example Delete job '10219837.oak-batch.osc.edu' from batch
+    #   my_conn.delete_job('10219837.oak-batch.osc.edu')
+    # @param id [#to_s] the id of the job
+    # @return [void]
+    def delete_job(id)
+      connect do |cid|
+        Torque.pbs_deljob cid, id.to_s, nil
+      end
+    end
+
+    # Submit a script to the batch server
+    # @example Submit a script with a few PBS directives
+    #   my_conn.submit_script("/path/to/script",
+    #     headers: {
+    #       Job_Name: "myjob",
+    #       Join_Path: "oe"
+    #     },
+    #     resources: {
+    #       nodes: "4:ppn=12",
+    #       walltime: "12:00:00"
+    #     },
+    #     envvars: {
+    #       TOKEN: "asd90f9sd8g90hk34"
+    #     }
+    #   )
+    #   #=> "6621251.oak-batch.osc.edu"
+    # @param script [#to_s] path to the script
+    # @param queue [#to_s] queue to submit script to
+    # @param headers [Hash] pbs headers
+    # @param resources [Hash] pbs resources
+    # @param envvars [Hash] pbs environment variables
+    # @param qsub [Boolean] whether use library or binary for submission
+    # @return [String] the id of the job that was created
+    def submit_script(script, queue: nil, headers: {}, resources: {}, envvars: {}, qsub: true)
+      send(qsub ? :qsub_submit : :pbs_submit, script, queue, headers, resources, envvars)
+    end
+
+    # Submit a script expanded into a string to the batch server
+    # @param string [#to_s] script as a string
+    # @param (see #submit_script)
+    # @return [String] the id of the job that was created
+    def submit_string(string, **kwargs)
+      Tempfile.open('qsub.') do |f|
+        f.write string.to_s
+        f.close
+        submit_script(f.path, **kwargs)
+      end
+    end
+
+    private
+      # Submit a script using Torque library
+      def pbs_submit(script, queue, headers, resources, envvars)
+        attribs = headers.dup
+        attribs[ATTR[:l]] = resources.dup unless resources.empty?
+        attribs[ATTR[:v]] = envvars.map{|k,v| "#{k}=#{v}"}.join(",") unless envvars.empty?
+
+        connect do |cid|
+          attropl = Torque::Attropl.from_hash attribs
+          Torque.pbs_submit cid, attropl, script.to_s, queue.to_s, nil
+        end
+      end
+
+      # Submit a script using Torque binary
+      # NB: The binary includes many useful filters and is preferred
+      def qsub_submit(script, queue, headers, resources, envvars)
+        params  = ["-q", "#{queue}@#{host}"]
+        params += resources.map{|k,v| ["-l", "#{k}=#{v}"]}.flatten unless resources.empty?
+        params += ["-v", envvars.map{|k,v| "#{k}=#{v}"}.join(",")] unless envvars.empty?
+        params += headers.map do |k,v|
+          if param = ATTR.key(k) and param.length == 1
+            ["-#{param}", "#{v}"]
+          else
+            ["-W", "#{k}=#{v}"]
+          end
+        end.flatten
+        params << script.to_s
+
+        o, e, s = Open3.capture3(prefix.join("bin", "qsub").to_s, *params)
+        raise PBS::Error, e unless s.success?
+        o.chomp
+      end
+  end
+end