ood_core 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb944d43beb0aced99e13efb2ef10bf33f9666c705c50ca5ae1727751de43073
-  data.tar.gz: 6e3cd66160be3bbd63124d6f2ddc794bce4ae64e385977828faba5dfd28ff838
+  metadata.gz: 3296708d7bc47f3379a9e4a6c845d3f25c5ccefb599f4b92406d9dffdaef220b
+  data.tar.gz: b6af9e90b67bc9a7a52203808d849d8800336b30b09bdb8ed204526d01bc92e9
 SHA512:
-  metadata.gz: 176e331a856c1e6958c444426d5c1b41aa881e90a69dca507b07f5463eb81355689e8391e0bf27823fc42a9484789f623ffd566b9d6c414c9cf741a7cafd1def
-  data.tar.gz: 15481101ad3120d3e8457612f2b8a8be4f1e268b38538b18b710f27887836f7a47eac3bf2e89d8f73745ae96ae78e21cd5ba5afefb4161cf95f435d6f2fdf001
+  metadata.gz: 623ac6e6f8081d68a3e925d1150c9f20a0f613ccfb6837519d1b95d04533a72caa403c54327aad85dcea9c0694cc23941f40307d942623c095f53fed7fc32026
+  data.tar.gz: 0d785a9ade36b2f6f62f9ae55672091346aa4fb76bf358e6c00d4bc007623b8d1798813474665fc7b4d850d89e041fae5c2fefc9719fbe9f53a161a76127eaad

data/CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.13.0] - 2020-08-10
+### Added
+- CloudyCluster CCQ Adapter
+
 ## [0.12.0] - 2020-08-05
 ### Added
 - qos option to Slurm and Torque [#205](https://github.com/OSC/ood_core/pull/205)
@@ -243,7 +247,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.12.0...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.13.0...HEAD
+[0.13.0]: https://github.com/OSC/ood_core/compare/v0.12.0...v0.13.0
 [0.12.0]: https://github.com/OSC/ood_core/compare/v0.11.4...v0.12.0
 [0.11.4]: https://github.com/OSC/ood_core/compare/v0.11.3...v0.11.4
 [0.11.3]: https://github.com/OSC/ood_core/compare/v0.11.2...v0.11.3

data/lib/ood_core/job/adapters/ccq.rb

@@ -0,0 +1,267 @@
+require "ood_core/job/adapters/helper"
+require "tempfile"
+
+module OodCore
+  module Job
+    class Factory
+      using Refinements::HashExtensions
+
+      # Build the Cloudy Cluster adapter from a configuration
+      # @param config [#to_h] the configuration for job adapter
+      # @option config [Object] :image (nil) The default VM image to use
+      # @option config [Object] :cloud (gcp) The cloud provider being used [gcp,aws]
+      # @option config [Object] :scheduler (nil) The name of the scheduler to use
+      # @option config [Object] :sge_root (nil) Path to SGE root, note that
+      # @option config [#to_h] :bin (nil) Path to CC client binaries
+      # @option config [#to_h] :bin_overrides ({}) Optional overrides to CC client executables
+      def self.build_ccq(config)
+        Adapters::CCQ.new(config.to_h.symbolize_keys)
+      end
+    end
+
+    module Adapters
+
+      class PromptError < StandardError; end
+
+      class CCQ < Adapter
+        using Refinements::ArrayExtensions
+
+        attr_reader :image, :cloud, :scheduler, :bin, :bin_overrides, :jobid_regex
+
+        def initialize(config)
+          @image = config.fetch(:image, nil)
+          @cloud = config.fetch(:cloud, gcp_provider)
+          @scheduler = config.fetch(:scheduler, nil)
+          @bin = config.fetch(:bin, '/opt/CloudyCluster/srv/CCQ')
+          @bin_overrides = config.fetch(:bin_overrides, {})
+          @jobid_regex = config.fetch(:jobid_regex, "job id is: (?<job_id>\\d+) you")
+        end
+
+        # Submit a job with the attributes defined in the job template instance
+        # @param script [Script] script object that describes the script and
+        #   attributes for the submitted job
+        # @param after [#to_s, Array<#to_s>] not used
+        # @param afterok [#to_s, Array<#to_s>] not used
+        # @param afternotok [#to_s, Array<#to_s>] not used
+        # @param afterany [#to_s, Array<#to_s>] not used
+        # @return [String] the job id returned after successfully submitting a
+        #   job
+        # @see Adapter#submit
+        def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
+          script_file = make_script_file(script.content)
+          args = []
+
+          # cluster configuration args
+          args.concat ["-s", scheduler] unless scheduler.nil?
+          args.concat [image_arg, image] unless image.nil?
+
+          args.concat ["-o", script.output_path.to_s] unless script.output_path.nil?
+          args.concat ["-e", script.error_path.to_s] unless script.error_path.nil?
+          args.concat ["-tl", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
+          args.concat ["-js", script_file.path.to_s]
+
+          args.concat script.native if script.native
+
+          output = call("ccqsub", args: args)
+          parse_job_id_from_ccqsub(output)
+        ensure
+          script_file.close
+        end
+
+        # Retrieve info for all jobs from the resource manager
+        # @return [Array<Info>] information describing submitted jobs
+        def info_all(attrs: nil)
+          args = []
+          args.concat ["-s", scheduler] unless scheduler.nil?
+
+          stat_output = call("ccqstat", args: args)
+          info_from_ccqstat(stat_output)
+        end
+
+        # Retrieve job info from the resource manager
+        # @param id [#to_s] the id of the job
+        # @return [Info] information describing submitted job
+        def info(id)
+          args = []
+          args.concat ["-s", scheduler] unless scheduler.nil?
+          args.concat ["-ji", id]
+
+          stat_output = call("ccqstat", args: args)
+
+          # WARNING: code path differs here than info_all because the output
+          # from ccqstat -ji $JOBID is much more data than just the 4
+          # columns that ccqstat gives.
+          info_from_ccqstat_extended(stat_output)
+        end
+
+        # Retrieve job status from resource manager
+        # @param id [#to_s] the id of the job
+        # @return [Status] status of job
+        # @see Adapter#status
+        def status(id)
+          info(id).status
+        end
+
+        # This adapter does not implement hold and will always raise
+        #   an exception.
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] always
+        # @return [void]
+        def hold(_)
+          raise NotImplementedError, "subclass did not define #hold"
+        end
+
+        # This adapter does not implement release and will always raise
+        #   an exception.
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] always
+        # @return [void]
+        def release(_)
+          raise NotImplementedError, "subclass did not define #release"
+        end
+
+        # Delete the submitted job
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def delete(id)
+          call("ccqdel", args: [id])
+        end
+
+        def directive_prefix
+          '#CC'
+        end
+
+        private
+
+        # Mapping of state codes
+        STATE_MAP =
+        {
+          'Error'         => :suspended,  # not running, but infrastructure still possibly exists
+          'CreatingCG'    => :queued,     # creating control group
+          'Pending'       => :queued,     # in queue
+          'Submitted'     => :queued,     #
+          'Provisioning'  => :queued,     # node is being provisioned
+          'Running'       => :running,    #
+          'Completed'     => :completed,  #
+        }.freeze
+
+        def gcp_provider
+          'gcp'
+        end
+
+        def aws_provider
+          'aws'
+        end
+
+        def image_arg
+          if cloud == gcp_provider
+            '-gcpgi'
+          else
+            '-awsami'
+          end
+        end
+
+        def call(cmd, args: [], env: {}, stdin: "")
+          cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
+          args = args.map(&:to_s)
+          env = env.to_h
+          o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin.to_s)
+          s.success? ? o : interpret_and_raise(e, cmd)
+        end
+
+        # helper function to interpret an error the command had given and
+        # raise a different error.
+        def interpret_and_raise(error, command)
+          # a special case with CCQ that prompts the user for username & password
+          # so let's be helpful and tell the user what to do.
+          if error.end_with?("EOFError: EOF when reading a line\n")
+            raise(
+              PromptError,
+              "The #{command} command was prompted. You need to generate the certificate " + 
+              "manually in a shell by running 'ccqstat'\nand entering your username/password"
+            )
+          else
+            raise(JobAdapterError, e.message)
+          end
+        end
+
+        # Convert seconds to duration
+        def seconds_to_duration(seconds)
+          format("%02d:%02d:%02d", seconds / 3600, seconds / 60 % 60, seconds % 60)
+        end
+
+        # helper to make a script file. We can't pipe it into ccq so we have to
+        # write a file.
+        def make_script_file(content)
+          file = Tempfile.new(tmp_file_name)
+          file.write(content.to_s)
+          file.flush
+          file
+        end
+
+        def tmp_file_name
+          'ccq_ood_script_'
+        end
+
+        def parse_job_id_from_ccqsub(output)
+          match_data = /#{jobid_regex}/.match(output)
+          # match_data could be nil, OR re-configured jobid_regex could be looking for a different named group
+          job_id = match_data&.named_captures&.fetch('job_id', nil)
+          throw JobAdapterError.new "Could not extract job id out of ccqsub output '#{output}'" if job_id.nil?
+          job_id
+        end
+
+        # parse an Ood::Job::Info object from extended ccqstat output
+        def info_from_ccqstat_extended(data)
+          raw = extended_data_to_hash(data)
+          data_hash = { native: raw }
+          data_hash[:status] = get_state(raw['status'])
+          data_hash[:id] = raw['name']
+          data_hash[:job_name] = raw['jobName']
+          data_hash[:job_owner] = raw['userName']
+          data_hash[:submit_host] = raw['submitHostInstanceId']
+          data_hash[:dispatch_time] = raw['startTime'].to_i
+          data_hash[:submission_time] = raw['dateSubmitted'].to_i
+          data_hash[:queue_name] = raw['criteriaPriority']
+
+          Info.new(data_hash)
+        end
+
+        # extended data is just lines of 'key: value' value, so parse
+        # it and stick it all in a hash.
+        def extended_data_to_hash(data)
+          Hash[data.to_s.scan(/(\w+): (\S+)/)]
+        end
+
+        def info_from_ccqstat(data)
+          infos = []
+
+          data.to_s.each_line do |line|
+            words = line.split(/\s/).reject(&:empty?)
+            next if !words.empty? && words[0] == "Id" # just skip the header
+
+            infos << Info.new(line_to_hash(words)) if words.size == 5
+          end
+
+          infos
+        end
+
+        def line_to_hash(words)
+          return unless words.size == 5
+
+          data_hash = {}
+          data_hash[:id] = words[0]
+          data_hash[:job_name] = words[1]
+          data_hash[:job_owner] = words[2]
+          data_hash[:status] = get_state(words[4])
+
+          data_hash
+        end
+
+        def get_state(state)
+          STATE_MAP.fetch(state, :undetermined)
+        end
+      end
+    end
+  end
+end

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-05 00:00:00.000000000 Z
+date: 2020-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -163,6 +163,7 @@ files:
 - lib/ood_core/errors.rb
 - lib/ood_core/invalid_cluster.rb
 - lib/ood_core/job/adapter.rb
+- lib/ood_core/job/adapters/ccq.rb
 - lib/ood_core/job/adapters/drmaa.rb
 - lib/ood_core/job/adapters/helper.rb
 - lib/ood_core/job/adapters/linux_host.rb