ood_core 0.28.0 → 0.29.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc94feaae3ffa1016b59b053179a18322a7e9c4ad1f937776864a2c2b19fc0a1
-  data.tar.gz: e41085ddcbbfff4a36723da314f41f1010e4db2fb28209ad781d5c2a6516cd46
+  metadata.gz: 53043c13d393367627c85321c8c7ef9d69d7a6cbab687ea32a82ab2077484024
+  data.tar.gz: e6c1f60a01e714e5ac02090a9c01ce7b6e96bde96d1568b31c60ab630cecc3c5
 SHA512:
-  metadata.gz: 85daa1b26fe2b973ecdbb3b9d2f72b37b142795f637d2db0a73611de0b4081c052b732c56e71ea7d37830bcf3211edc6c01109a2d2514b1a2500b38ac9ac6b98
-  data.tar.gz: 3a2f47afde0ac909f4d3ea6a7ccf2fcf48b8e13de94ad4e550ee1b2492cf28c7dcc72df179755b332b358f49accb72e8c229a1b506de6614148db14233e6c902
+  metadata.gz: a5ee3699f5737abf3158e00341fd7e236b6320893f9a3640f7e5b61c854796512e03277c699db100ebeef402f735a58e82596b50dce152a381b01f43cb93ce3b
+  data.tar.gz: 5ff0fc3ed64f3154394aeedc5caed25fd0779304963dbbecf0b6b00083185459abfa40a3787da43f0da21eb475c30bb726c26514e7395ece23b193081e4f2cb1

data/lib/ood_core/job/adapters/htcondor.rb

@@ -0,0 +1,549 @@
+require "time"
+require 'etc'
+require 'tempfile'
+require "ood_core/refinements/hash_extensions"
+require "ood_core/refinements/array_extensions"
+require "ood_core/job/adapters/helper"
+
+module OodCore
+    module Job
+        class Factory
+            using Refinements::HashExtensions
+
+            # Build the HTCondor adapter from a configuration
+            # @param config [#to_h] the configuration for job adapter
+            # @option config [Object] :bin (nil) Path to HTCondor client binaries
+            # @option config [Object] :submit_host ("") Submit job on login node via ssh
+            # @option config [Object] :strict_host_checking (true) Whether to use strict host checking when ssh to submit_host
+            def self.build_htcondor(config)
+                c = config.to_h.symbolize_keys
+                bin                  = c.fetch(:bin, nil)
+                bin_overrides        = c.fetch(:bin_overrides, {})
+                submit_host          = c.fetch(:submit_host, "")
+                strict_host_checking = c.fetch(:strict_host_checking, true)
+                default_universe = c.fetch(:default_universe, "vanilla")
+                default_docker_image = c.fetch(:default_docker_image, "ubuntu:latest")
+                user_group_map = c.fetch(:user_group_map, nil)
+                cluster = c.fetch(:cluster, "")
+                additional_attributes = c.fetch(:additional_attributes, {})
+                htcondor = Adapters::HTCondor::Batch.new(bin: bin, bin_overrides: bin_overrides,
+                    submit_host: submit_host, strict_host_checking: strict_host_checking,
+                    default_universe: default_universe,
+                    default_docker_image: default_docker_image,
+                    user_group_map: user_group_map,
+                    cluster: cluster,
+                    additional_attributes: additional_attributes,
+                    )
+                Adapters::HTCondor.new(htcondor: htcondor)
+            end
+        end
+
+        module Adapters
+            # An adapter object that describes the communication with an HTCondor
+            # resource manager for job management.
+            class HTCondor < Adapter
+                using Refinements::HashExtensions
+                using Refinements::ArrayExtensions
+
+                # Object used for simplified communication with an HTCondor batch server
+                # @api private
+                class Batch
+                    # The path to the HTCondor client installation binaries
+                    # @return [Pathname] path to HTCondor binaries
+                    attr_reader :bin
+
+                    # The path to the HTCondor client installation binaries that override
+                    # the default binaries
+                    # @return [Pathname] path to HTCondor binaries overrides
+                    attr_reader :bin_overrides
+
+                    # The login node where the job is submitted via ssh
+                    # @return [String] The login node
+                    attr_reader :submit_host
+
+                    # Whether to use strict host checking when ssh to submit_host
+                    # @return [Bool]; true if empty
+                    attr_reader :strict_host_checking
+
+                    # Default universe for jobs submitted to HTCondor
+                    # @return [String] the default universe for jobs
+                    attr_reader :default_universe
+
+                    # Default docker image for jobs submitted to HTCondor
+                    # @return [String] the default docker image for jobs
+                    attr_reader :default_docker_image
+
+                    # A path to the user/group map for HTCondor jobs
+                    # The format in the file should adhere to the format used by [AssignAccountingGroup](https://htcondor.readthedocs.io/en/latest/admin-manual/introduction-to-configuration.html#FEATURE:ASSIGNACCOUNTINGGROUP)
+                    # @return [String,nil] the path to the user/group map file
+                    attr_reader :user_group_map
+
+                    # The cluster name for this HTCondor instance
+                    # @return [String] the cluster name
+                    attr_reader :cluster
+
+                    # Additional attributes to be added to the job submission
+                    # @return [Hash{#to_s => #to_s}] additional attributes to be added to the job submission
+                    attr_reader :additional_attributes
+
+                    # The version of HTCondor on the submit_host
+                    # @return [Gem::Version] the version of HTCondor
+                    attr_reader :version
+
+                    # The root exception class that all HTCondor-specific exceptions inherit
+                    # from
+                    class Error < StandardError; end
+
+                    # @param bin [#to_s] path to HTCondor installation binaries
+                    # @param submit_host [#to_s] Submits the job on a login node via ssh
+                    # @param strict_host_checking [Bool] Whether to use strict host checking when ssh to submit_host
+                    def initialize(bin: nil, bin_overrides: {}, submit_host: "", strict_host_checking: false, default_universe: "vanilla", default_docker_image: "ubuntu:latest", user_group_map: nil, cluster: "", additional_attributes: {})
+                        @bin                  = Pathname.new(bin.to_s)
+                        @bin_overrides        = bin_overrides
+                        @submit_host          = submit_host.to_s
+                        @strict_host_checking = strict_host_checking
+                        @default_universe     = default_universe.to_s
+                        @default_docker_image = default_docker_image.to_s
+                        @user_group_map       = user_group_map.to_s unless user_group_map.nil?
+                        @cluster              = cluster.to_s
+                        @additional_attributes = additional_attributes
+                        @version = get_htcondor_version
+                    end
+
+                    # Submit a script to the batch server
+                    # @param args [Array<#to_s>] arguments passed to `condor_submit` command
+                    # @param env [Hash{#to_s => #to_s}] environment variables set
+                    # @param script [String] the script to submit
+                    # @raise [Error] if `condor_submit` command exited unsuccessfully
+                    # @return [String] the id of the job that was created
+                    def submit_string(args: [], script_args: [], env: {}, script: "")
+                        args = args.map(&:to_s)
+                        script_args = script_args.map(&:to_s).map { |s| s.to_s.gsub('"', "'") } # cannot do double
+                        env = env.to_h.each_with_object({}) { |(k, v), h| h[k.to_s] = v.to_s }
+
+                        path = "#{Dir.tmpdir}/htcondor_submit_#{SecureRandom.uuid}"
+
+                        call("bash", "-c", "cat > #{path}", stdin: script)
+                        output = call("condor_submit", *args, env: env, stdin: "arguments=#{path.split("/").last} #{script_args.join(" ")}\ntransfer_input_files=#{path}").strip
+
+                        match = output.match(/(cluster )?(\d+)/)
+                        raise Error, "Failed to parse job ID from output: #{output}" unless match
+                        match[2]
+
+                    end
+
+                    # Run the `condor_rm` command to remove a job
+                    # @param id [#to_s] the id of the job to remove
+                    # @raise [Error] if `condor_rm` command exited unsuccessfully
+                    def remove_job(id)
+                        call("condor_rm", id.to_s)
+                    rescue Error => e
+                        raise Error, "Failed to remove job #{id}: #{e.message}"
+                    end
+
+                    # Place a job on hold using `condor_hold`
+                    # @param id [#to_s] the id of the job to hold
+                    # @raise [Error] if `condor_hold` command exited unsuccessfully
+                    def hold_job(id)
+                        id = id.to_s
+                        call("condor_hold", id)
+                    rescue Error => e
+                        raise Error, "Failed to hold job #{id}: #{e.message}"
+                    end
+
+                    # Release a job from hold using `condor_release`
+                    # @param id [#to_s] the id of the job to release
+                    # @raise [Error] if `condor_release` command exited unsuccessfully
+                    def release_job(id)
+                        id = id.to_s
+                        call("condor_release", id)
+                    rescue Error => e
+                        raise Error, "Failed to release job #{id}: #{e.message}"
+                    end
+
+                    def condor_q_attrs
+                        {
+                            id: "ClusterId",
+                            sub_id: "ProcId",
+                            status: "JobStatus",
+                            owner: "Owner",
+                            acct_group: "AcctGroup",
+                            job_name: "JobBatchName",
+                            procs: "CpusProvisioned",
+                            gpus: "GpusProvisioned",
+                            submission_time: "QDate",
+                            dispatch_time: "JobCurrentStartDate",
+                            sys_cpu_time: "RemoteSysCpu",
+                            user_cpu_time: "RemoteUserCpu",
+                            wallclock_time: "RemoteWallClockTime"
+                        }
+                    end
+
+                    # Retrieve job information using `condor_q`
+                    # @param id [#to_s] the id of the job
+                    # @param owner [String] the owner(s) of the job
+                    # @raise [Error] if `condor_q` command exited unsuccessfully
+                    # @return [Array<Hash>] list of details for jobs
+                    def get_jobs(id: "", owner: nil)
+                        args = []
+                        unless id.to_s.empty?
+                            if id.to_s.include?(".") # if id is a job array, we need to use the ClusterId and ProcId
+                                cluster_id, proc_id = id.to_s.split(".")
+                                args.concat ["-constraint", "\"ClusterId == #{cluster_id} && ProcId == #{proc_id}\""]
+                            else # if id is a single job, we can just use the ClusterId
+                                args.concat ["-constraint", "\"ClusterId == #{id}\""]
+                            end
+                        end
+                        args.concat ["-constraint", "\"Owner == #{owner}\""] unless owner.to_s.empty?
+                        args.concat ["-af", *condor_q_attrs.values]
+
+                        output = call("condor_q", *args)
+                        parse_condor_q_output(output)
+                    end
+
+                    # Retrieve slot information using `condor_status`
+                    # @param owner [String] the owner(s) of the slots
+                    # @raise [Error] if `condor_status` command exited unsuccessfully
+                    # @return [Array<Hash>] list of details for slots
+                    def get_slots
+                        args = ["-af", "Machine", "TotalSlotCPUs", "TotalSlotGPUs", "TotalSlotMemory", "CPUs", "GPUs", "Memory", "NumDynamicSlots"]
+                        args.concat ["-constraint", "\"DynamicSlot is undefined\""]
+
+                        output = call("condor_status", *args)
+                        parse_condor_status_output(output)
+                    end
+
+
+                    # Retrieve accounts using user_group_map on @submit_host
+                    # @return [Hash{String => Array<String>}] mapping of usernames to their groups
+                    def get_accounts
+                        raise Error, "user_group_map is not defined" if user_group_map.nil? || user_group_map.empty?
+
+                        # Retrieve accounts, use local file, if exists. Otherwise use from submit_host
+                        if File.exist?(user_group_map) && File.readable?(user_group_map)
+                            output = File.read(user_group_map)
+                        else
+                            output = call("cat", user_group_map)
+                        end
+                        accounts = {}
+                        output.each_line do |line|
+                            next if line.strip.empty? || line.start_with?("#") # Skip empty lines and comments
+                            _, username, groups = line.strip.split(/\s+/, 3)
+                            accounts[username] = groups.split(",") if username && groups
+                        end
+
+                        accounts
+                    rescue Error => e
+                        raise Error, "Failed to retrieve accounts: #{e.message}"
+                    end
+
+                    private
+
+                    # Parse the output of `condor_q` into a list of job hashes
+                    def parse_condor_q_output(output)
+                        jobs = []
+                        fields = condor_q_attrs
+                        output.each_line do |line|
+                            # Parse each line into a hash
+                            job_data = line.split
+                            job = Hash[fields.keys.zip(job_data)]
+                            job[:submit_host] = @submit_host # Add submit host to job data
+                            job[:native] = job_data # Add native attributes to job data
+                            jobs << job
+                        end
+                        jobs
+                    end
+
+                    # Parse the output of `condor_status` into a list of slot hashes
+                    def parse_condor_status_output(output)
+                        slots = []
+                        output.each_line do |line|
+                            # Parse each line into a hash (custom parsing logic for HTCondor slots)
+                            slot_data = line.split
+                            slots << { machine: slot_data[0], total_cpus: slot_data[1].to_i, total_gpus: slot_data[2].to_i, total_memory: slot_data[3].to_i,
+                                       cpus: slot_data[4].to_i, gpus: slot_data[5].to_i, memory: slot_data[6].to_i,
+                                       num_dynamic_slots: slot_data[7].to_i }
+                        end
+                        slots
+                    end
+
+                    # Call a forked HTCondor command
+                    def call(cmd, *args, env: {}, stdin: "")
+                        cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
+                        args = args.map(&:to_s)
+
+                        cmd, args = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, args, strict_host_checking)
+                        o, e, s = Open3.capture3(env, cmd, *(args.map(&:to_s)), stdin_data: stdin.to_s)
+                        s.success? ? o : raise(Error, e)
+                    end
+
+                    def get_htcondor_version
+                        output = call("condor_version")
+                        match = output.match(/CondorVersion: (\d+\.\d+\.\d+)/)
+                        raise Error, "Failed to parse HTCondor version from output: #{output}" unless match
+                        Gem::Version.new(match[1])
+                    end
+                end
+
+                # Map HTCondor job statuses to symbols
+                STATUS_MAP = {
+                    "1" => :queued,
+                    "2" => :running,
+                    "3" => :running,
+                    "4" => :completed,
+                    "5" => :queued_held,
+                    "6" => :running,
+                    "7" => :suspended
+                }.freeze
+
+                # @api private
+                # @param opts [#to_h] the options defining this adapter
+                # @option opts [Batch] :htcondor The HTCondor batch object
+                # @see Factory.build_htcondor
+                def initialize(opts = {})
+                    o = opts.to_h.symbolize_keys
+
+                    @htcondor = o.fetch(:htcondor) { raise ArgumentError, "No HTCondor object specified. Missing argument: htcondor" }
+                end
+
+                # Submit a job with the attributes defined in the job template instance
+                # @param script [Script] script object that describes the script and
+                #   attributes for the submitted job
+                # @raise [JobAdapterError] if something goes wrong submitting a job
+                # @return [String] the job id returned after successfully submitting a
+                #   job
+                def submit(script)
+                    args = []
+                    args.concat ["-batch-name", "#{script.job_name}"] unless script.job_name.nil?
+                    args.concat ["-name", "#{script.queue_name}"] unless script.queue_name.nil?
+                    args.concat ["-a", "priority=#{script.priority}"] unless script.priority.nil?
+                    args.concat ["-a", "accounting_group=#{script.accounting_id}"] unless script.accounting_id.nil?
+
+                    args.concat ["-a", "submit_as_hold=#{script.hold}"] unless script.submit_as_hold.nil?
+                    args.concat ["-a", "max_retries=0"] unless !script.rerunnable.nil? && script.rerunnable
+
+                    args.concat ["-a", "allowed_execute_duration=#{script.wall_time}"] unless script.wall_time.nil?
+                    args.concat ["-a", "periodic_remove='HoldReasonCode == 47'"] unless script.wall_time.nil?
+                    args.concat ["-a", "deferral_time=#{script.start_time.tv_sec}"] unless script.start_time.nil?
+
+                    args.concat ["-a", "request_cpus=#{script.cores}"] unless script.cores.nil?
+                    # requesting 1GB of memory per core seems reasonable
+                    args.concat ["-a", "request_memory=#{script.cores * 1024}"] unless script.native.include?(:request_memory) && !script.native[:request_memory].nil?
+                    args.concat ["-a", "request_gpus=#{script.gpus_per_node}"] unless script.gpus_per_node.nil?
+
+                    universe = script.native[:universe] || @htcondor.default_universe
+                    args.concat ["-a", "universe=#{universe}"]
+                    container_image = script.native[:docker_image] || @htcondor.default_docker_image
+                    if universe == "docker" then
+                        args.concat ["-a", "docker_image=#{@htcondor.default_docker_image}"] unless script.native.include?(:docker_image) && !script.native[:docker_image].nil?
+                    elsif universe == "container" then
+                        script.native.delete(:docker_image) unless !script.native.include?(:docker_image)
+                        script.native[:container_image] = container_image
+                    end
+
+                    args.concat ["-a", "input=#{script.input_path}"] unless script.input_path.nil?
+                    if script.output_path.nil? then args.concat ["-a", "output=output.txt"] else args.concat ["-a", "output=#{script.output_path}"] end
+                    if script.error_path.nil? then args.concat ["-a", "error=error.txt"] else args.concat ["-a", "error=#{script.error_path}"] end
+                    if script.workdir.nil? then args.concat ["-a", "log=job.log"] else args.concat ["-a", "log=#{script.workdir}/job.log"] end
+
+                    args.concat ["-a", "initialdir=#{script.workdir}"] unless script.workdir.nil?
+                    args.concat ["-a", "\"environment=\\\"#{script.job_environment.to_a.map { |k, v| "#{k}='#{v.gsub("'", "''").gsub('"', "\\\"\\\"")}'" }.join(' ')}\\\"\""] unless script.job_environment.nil? || script.job_environment.empty?
+                    args.concat ["-a", "getenv=#{script.copy_environment}"] unless script.copy_environment.nil?
+
+                    args.concat ["-a", "should_transfer_files=true"]
+                    args.concat ["-a", "+OpenOnDemand=true"]
+
+                    # send email when started / terminated
+                    if script.email_on_started && script.email_on_terminated then
+                        raise JobAdapterError, "Cannot handle both email_on_started and email_on_terminated set to true" if script.email_on_started && script.email_on_terminated
+                        # args.concat ["-a", "notification=Always"] # might be supported in the future?
+                    elsif script.email_on_started then
+                        if @htcondor.version >= Gem::Version.new("24.10.0") then
+                            args.concat ["-a", "notification=Start"]
+                        else
+                            raise JobAdapterError, "Email notification on job start is not supported by this HTCondor version. Please upgrade to 24.10.0 or later."
+                        end
+                    elsif script.email_on_terminated then
+                        args.concat ["-a", "notification=Complete"]
+                    else
+                        args.concat ["-a", "notification=Never"]
+                    end
+                    args.concat ["-a", "notify_user=#{script.email}"] unless script.email.nil?
+
+                    args.concat @htcondor.additional_attributes.to_a.map { |k, v| "-a #{k}=#{v}" } unless @htcondor.additional_attributes.nil? || @htcondor.additional_attributes.empty?
+                    args.concat script.native.to_a.map { |k, v| "-a #{k}=#{v}" } unless script.native.nil? || script.native.empty?
+
+                    content = script.content
+
+                    # Set executable to some shell to execute the script
+                    if script.shell_path.nil?
+                        args.concat ["-a", "executable=/bin/bash"]
+                    else
+                        args.concat ["-a", "executable=#{script.shell_path}"]
+                    end
+
+                    # terse to shut up the output, - to get the script arguments from stdin.
+                    args.concat ["-terse", "-"]
+
+                    if script.job_array_request.nil?
+                        # If no job array request is specified, we submit a single job
+                        args.concat ["-queue", "1"]
+                    else
+                        # If a job array request is specified, we submit a job array
+                        # The job array request is expected to be a string like "1-10" or "1,2,3"
+                        # we must convert 1-3 to 1,2,3.
+                        if script.job_array_request.include?("-")
+                            start, finish = script.job_array_request.split("-").map(&:to_i)
+                            job_ids = (start..finish).to_a.join(",")
+                        else
+                            job_ids = script.job_array_request
+                        end
+                        # Generate multiple jobs in the job array by setting OODArrayId to the requested array ids
+                        # While -queue 10 would generate 10 jobs, the ProcId would always be 0-9, not 1-10 - or whatever the request is.
+                        # So we set the OODArrayId to the requested job ids.
+                        args.concat ["-queue", "1", "+OODArrayId", "in", job_ids.to_s]
+                    end
+
+                    script_args = script.args || []
+
+                    @htcondor.submit_string(args: args, script_args: script_args, script: content)
+                rescue Batch::Error => e
+                    raise JobAdapterError, e.message
+                end
+
+                # Retrieve job info from the resource manager
+                # @param id [#to_s] the id of the job
+                # @raise [JobAdapterError] if something goes wrong getting job info
+                # @return [Info] information describing submitted job
+                def info(id)
+                    id = id.to_s
+                    jobs = @htcondor.get_jobs(id: id)
+                    jobs.empty? ? Info.new(id: id, status: :completed) : parse_job_info(jobs.first)
+                rescue Batch::Error => e
+                    raise JobAdapterError, e.message
+                end
+
+                # Retrieve information for all jobs
+                # @raise [JobAdapterError] if something goes wrong retrieving job info
+                # @return [Array<Info>] list of information describing submitted jobs
+                def info_all(attrs: nil)
+                    jobs = @htcondor.get_jobs
+                    jobs.map { |job| parse_job_info(job) }
+                rescue Batch::Error => e
+                    raise JobAdapterError, e.message
+                end
+
+                # Retrieve the status of a job
+                # @param id [#to_s] the id of the job
+                # @raise [JobAdapterError] if something goes wrong retrieving the job status
+                # @return [Symbol] the status of the job
+                def status(id)
+                    id = id.to_s
+                    jobs = @htcondor.get_jobs(id: id)
+                    jobs.empty? ? :completed : get_state(jobs.first[:status])
+                rescue Batch::Error => e
+                    raise JobAdapterError, e.message
+                end
+
+                # Retrieve cluster status information
+                # @raise [JobAdapterError] if something goes wrong retrieving cluster status
+                # @return [Hash] summary of cluster status including active and total nodes, processors, GPUs, etc.
+                def cluster_info
+                    slots = @htcondor.get_slots
+                    active_nodes = slots.count { |slot| slot[:num_dynamic_slots] > 0 }
+                    total_nodes = slots.map { |slot| slot[:machine] }.uniq.count
+                    active_processors = slots.sum { |slot| slot[:total_cpus] - slot[:cpus] }
+                    total_processors = slots.sum { |slot| slot[:total_cpus] }
+                    active_gpus = slots.sum { |slot| slot[:total_gpus] - slot[:gpus] }
+                    total_gpus = slots.sum { |slot| slot[:total_gpus] }
+
+                    ClusterInfo.new({
+                        active_nodes: active_nodes,
+                        total_nodes: total_nodes,
+                        active_processors: active_processors,
+                        total_processors: total_processors,
+                        active_gpus: active_gpus,
+                        total_gpus: total_gpus
+                    })
+                rescue Batch::Error => e
+                    raise JobAdapterError, e.message
+                end
+
+                # Indicate that the job adapter supports job arrays
+                def supports_job_arrays?
+                    true
+                end
+
+                # Place a job on hold
+                # @param id [#to_s] the id of the job
+                # @raise [JobAdapterError] if something goes wrong placing the job on hold
+                def hold(id)
+                    @htcondor.hold_job(id)
+                rescue Batch::Error => e
+                    raise JobAdapterError, e.message
+                end
+
+                # Release a job from hold
+                # @param id [#to_s] the id of the job
+                # @raise [JobAdapterError] if something goes wrong releasing the job
+                def release(id)
+                    @htcondor.release_job(id)
+                rescue Batch::Error => e
+                    raise JobAdapterError, e.message
+                end
+                # Delete a job
+                # @param id [#to_s] the id of the job
+                # @raise [JobAdapterError] if something goes wrong deleting the job
+                def delete(id)
+                    @htcondor.remove_job(id)
+                rescue Batch::Error => e
+                    raise JobAdapterError, e.message
+                end
+
+                # Retrieve the relevant groups for the current user
+                # @return [Array<AccountInfo>] list of groups for the current user
+                def accounts
+                    username = Etc.getlogin
+                    groups = @htcondor.get_accounts[username]
+                    parse_group_into_account_info(groups)
+                rescue Batch::Error => e
+                    raise JobAdapterError, e.message
+                end
+
+                private
+
+                def get_state(st)
+                    STATUS_MAP.fetch(st.to_s, :undetermined)
+                end
+
+                # Parse hash describing HTCondor job status
+                def parse_job_info(job)
+                    Info.new(
+                        id: job[:id].to_s + (job[:sub_id].to_s.empty? ? "" : ".#{job[:sub_id]}"),
+                        status: get_state(job[:status]),
+                        job_name: job[:job_name],
+                        job_owner: job[:owner],
+                        accounting_id: job[:acct_group],
+                        submit_host: job[:submit_host],
+                        procs: job[:procs].to_i,
+                        gpus: job[:gpus].to_i,
+                        submission_time: Time.at(job[:submission_time].to_i),
+                        dispatch_time: Time.at(job[:dispatch_time].to_i),
+                        cpu_time: job[:sys_cpu_time].to_i + job[:user_cpu_time].to_i,
+                        wallclock_time: job[:wallclock_time].to_i,
+                        native: job[:native],
+
+                    )
+                end
+
+                # Parse group information into AccountInfo objects
+                # @param groups [Array<String>] list of group names
+                # @return [Array<AccountInfo>] list of AccountInfo objects
+                def parse_group_into_account_info(groups)
+                    groups.map { |group| AccountInfo.new(name: group, cluster: @htcondor.cluster) }
+                end
+
+            end
+        end
+    end
+end

data/lib/ood_core/job/adapters/psij/delete.py

@@ -0,0 +1,18 @@
+import argparse
+
+parser = argparse.ArgumentParser(description="Process job parameters")
+parser.add_argument("--id", type=str, required=True, help="Path to the job script")
+parser.add_argument("--executor", type=str, required=True, help="Executor to be used")
+
+args = parser.parse_args()
+
+from psij import Job, JobExecutor
+
+ex = JobExecutor.get_instance(args.executor)
+job = Job()
+job._native_id = args.id
+# catch exception
+try:
+  ex.cancel(job)
+except Exception as e:
+  print(f"Invalid job id specified")

data/lib/ood_core/job/adapters/psij/get_info.py

@@ -0,0 +1,55 @@
+import argparse
+import json
+from datetime import datetime, timedelta
+import time
+
+parser = argparse.ArgumentParser(description="Process job parameters")
+parser.add_argument("--id", type=str, help="Path to the job script")
+parser.add_argument("--owner", type=str, help="the name of job owner")
+parser.add_argument("--executor", type=str, required=True, help="Executor to be used")
+
+args = parser.parse_args()
+
+from psij import Job, JobExecutor
+from psij.serialize import JSONSerializer
+
+ex = JobExecutor.get_instance(args.executor)
+if args.id:
+    job = Job()
+    job._native_id = args.id
+    job_data = ex.info([job])
+elif args.owner:
+    job_data = ex.info(owner=args.owner)
+else:
+    job_data = ex.info()
+
+s = JSONSerializer()
+# create dict for each job.
+# [ {'native_id': native_id, ... }, {'native_id': native_id, ...}, ...]
+data = []
+for job in job_data:
+    d = {}
+    d["native_id"] = job.native_id
+    d["current_state"] = job._status.state.name
+    d.update(job.current_info.__dict__)
+    d.update(s._from_spec(job.spec))
+    # the attributes and resources are nested in the job data.
+    # we need to flatten them.
+    attr = d["attributes"]
+    del d["attributes"]
+    d.update(attr)
+    # convert deltatime or string to integer
+    d["duration"] = job.spec.attributes.duration.total_seconds()
+    d["wall_time"] = int(d["wall_time"])
+    resources = d["resources"]
+    del d["resources"]
+    d.update(resources)
+    d["submission_time"] = d["submission_time"].strftime("%Y-%m-%d %H:%M:%S")
+    if d["dispatch_time"] is not None:
+      d["dispatch_time"] = d["dispatch_time"].strftime("%Y-%m-%d %H:%M:%S")
+    else:
+      d["dispatch_time"] = None
+
+    data.append(d)
+
+print(json.dumps(data))

data/lib/ood_core/job/adapters/psij/hold.py

@@ -0,0 +1,18 @@
+import argparse
+
+parser = argparse.ArgumentParser(description="Process job parameters")
+parser.add_argument("--id", type=str, required=True, help="Path to the job script")
+parser.add_argument("--executor", type=str, required=True, help="Executor to be used")
+
+args = parser.parse_args()
+
+from psij import Job, JobExecutor
+
+ex = JobExecutor.get_instance(args.executor)
+job = Job()
+job._native_id = args.id
+# catch exception
+try:
+  ex.hold(job)
+except Exception as e:
+  print(f"Invalid job id specified")

data/lib/ood_core/job/adapters/psij/release.py

@@ -0,0 +1,18 @@
+import argparse
+
+parser = argparse.ArgumentParser(description="Process job parameters")
+parser.add_argument("--id", type=str, required=True, help="Path to the job script")
+parser.add_argument("--executor", type=str, required=True, help="Executor to be used")
+
+args = parser.parse_args()
+
+from psij import Job, JobExecutor
+
+ex = JobExecutor.get_instance(args.executor)
+job = Job()
+job._native_id = args.id
+# catch exception
+try:
+  ex.release(job)
+except Exception as e:
+  print(f"Invalid job id specified")

data/lib/ood_core/job/adapters/psij/submit.py

@@ -0,0 +1,28 @@
+import sys
+from psij import Job, JobExecutor
+from psij.serialize import JSONSerializer
+from pathlib import Path
+import json
+import os
+
+# create executor instance.
+ex = JobExecutor.get_instance(sys.argv[1])
+
+# deserialize json data to job spec.
+deserialize = JSONSerializer()
+d = sys.stdin.read()
+j = json.loads(d)
+spec = deserialize._to_spec(j)
+
+# add executor string to each key of custom attributes.
+if sys.argv[1] != "local" and spec.attributes.custom_attributes is not None:
+    h = {}
+    for k in spec.attributes.custom_attributes.keys():
+        h[f"{ex.name}.{k}"] = spec.attributes.custom_attributes[k]
+    spec.attributes.custom_attributes = h
+
+spec.executable = os.path.expanduser(spec.executable)
+job = Job(spec)
+
+ex.submit(job)
+print(job.native_id)

data/lib/ood_core/job/adapters/psij.rb

@@ -0,0 +1,410 @@
+require "time"
+require 'etc'
+require "ood_core/refinements/hash_extensions"
+require "ood_core/refinements/array_extensions"
+require "ood_core/job/adapters/helper"
+
+require 'json'
+require 'pathname'
+
+module OodCore
+  module Job
+		class Factory
+ 
+      using Refinements::HashExtensions
+			# Build the PSIJ adapter from a configuration
+      # @param config [#to_h] the configuration for job adapter
+      # @option config [Object] :bin (nil) Path to PSIJ binaries
+      # @option config [#to_h]  :bin_overrides ({}) Optional overrides to PSIJ executables
+			def self.build_psij(config)
+        c = config.to_h.symbolize_keys
+        cluster              = c.fetch(:cluster, nil)
+        conf                 = c.fetch(:conf, nil)
+        bin                  = c.fetch(:bin, nil)
+        bin_overrides        = c.fetch(:bin_overrides, {})
+        submit_host          = c.fetch(:submit_host, "")
+        strict_host_checking = c.fetch(:strict_host_checking, true)
+        executor             = c.fetch(:executor, nil)
+        queue_name           = c.fetch(:queue_name, nil)
+        psij = Adapters::PSIJ::Batch.new(cluster: cluster, conf: conf, bin: bin, bin_overrides: bin_overrides, submit_host: submit_host, strict_host_checking: strict_host_checking, executor: executor, queue_name: queue_name)
+        Adapters::PSIJ.new(psij: psij)
+      end
+		end
+
+		module Adapters
+      class PSIJ < Adapter
+        using Refinements::HashExtensions
+        using Refinements::ArrayExtensions
+        class Batch
+
+          attr_reader :cluster
+          attr_reader :conf
+          attr_reader :bin
+          attr_reader :bin_overrides
+          attr_reader :submit_host
+          attr_reader :strict_host_checking
+          attr_reader :executor
+          attr_reader :queue_name
+
+          class Error < StandardError; end
+
+          def initialize(cluster: nil, bin: nil, conf: nil, bin_overrides: {}, submit_host: "", strict_host_checking: true, executor: nil, queue_name: nil)
+            @cluster              = cluster && cluster.to_s
+            @conf                 = conf    && Pathname.new(conf.to_s)
+            @bin                  = Pathname.new(bin.to_s)
+            @bin_overrides        = bin_overrides
+            @submit_host          = submit_host.to_s
+            @strict_host_checking = strict_host_checking
+            @executor             = executor
+            @queue_name           = queue_name
+          end
+
+          def get_jobs(id: "", owner: nil)
+            id = id.to_s.strip()
+            params = {
+              id: id,
+              executor: executor,
+            }
+            args = params.map { |k, v| "--#{k}=#{v}" }
+            get_info_path = Pathname.new(__FILE__).dirname.expand_path.join("psij/get_info.py").to_s
+            jobs_data = call("python3", get_info_path, *args)
+            jobs_data = JSON.parse(jobs_data, symbolize_names: true)
+            jobs_data
+          end
+
+          def submit_job_path(args: [], chdir: nil, stdin: nil)
+            submit_path = Pathname.new(__FILE__).dirname.expand_path.join("psij/submit.py").to_s
+            call("python3", submit_path, *args, chdir: chdir, stdin: stdin)
+          end
+
+          def delete_job(args: [])
+            delete_path = Pathname.new(__FILE__).dirname.expand_path.join("psij/delete.py").to_s
+            call("python3", delete_path, *args)
+          rescue => e
+            raise JobAdapterError, e
+          end
+
+          def hold_job(args: [])
+            hold_path = Pathname.new(__FILE__).dirname.expand_path.join("psij/hold.py").to_s
+            call("python3", hold_path, *args)
+          end
+
+          def release_job(args: [])
+            release_path = Pathname.new(__FILE__).dirname.expand_path.join("psij/release.py").to_s
+            call("python3", release_path, *args)
+          end
+
+          def seconds_to_duration(time)
+            "%02d:%02d:%02d" % [time/3600, time/60%60, time%60]
+          end
+
+          private
+            # Call a forked psij script for a given cluster
+            def call(cmd, *args, env: {}, stdin: "", chdir: nil)
+              cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
+              cmd, args = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, args, strict_host_checking)
+              chdir ||= "."
+              o, e, s = Open3.capture3(env, cmd, *(args.map(&:to_s)), stdin_data: stdin, chdir: chdir.to_s)
+              s.success? ? o : raise(Error, e)
+            end
+
+        end
+        
+
+        STATE_MAP = {
+          'NEW'           => :undetermined,
+          'QUEUED'        => :queued,
+          'HELD'          => :queued_held,
+          'ACTIVE'        => :running,
+          'COMPLETED'     => :completed,
+        }
+
+        def initialize(opts = {})
+          o = opts.to_h.symbolize_keys
+        
+          @psij = o.fetch(:psij) { raise ArgumentError, "No psij object specified. Missing argument: psij" }
+        end
+        
+
+        # The `submit` method saves a job script as a file and prepares a command to submit the job.
+        # Each optional argument specifies job dependencies (after, afterok, afternotok, afterany).
+        def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
+          # convert OOD interfaces to PSI/J interfaces.
+          # Conterted variables are shown as follows:
+          #       OOD           |   PSI/J(JobSpec)
+          # --------------------+----------------------------------------------------
+          # submit_as_hold      |       X (not support)
+          # rerunnable          |       X
+          # email_on_started    |       X
+          # email_on_terminated |       X
+          # args                |   JobAttributes.custom_attributes
+          # job_environment     |   environment
+          # workdir             |   directory
+          # email               |       X
+          # job_name            |   name
+          # shell_path          |   #!<shell_path>
+          # input_path          |   stdin_path
+          # output_path         |   stdout_path
+          # error_path          |   stderr_path
+          # reservation_id      |   JobAttributes.reservation_id
+          # queue_name          |   JobAttributes.queue_name
+          # priority            |       X
+          # start_time          |       X
+          # wall_time           |   JobAttributes.duration
+          # accounting_id       |   JobAttributes.account or project_name(duplicated)
+          # job_array_request   |       X
+          # qos                 |       X
+          # gpus_per_node       |   ResourceSpec.gpu_cores_per_process
+          # native              |   executable (join script.content)
+          # copy_environment    |   inherit_envrionment
+          # cores               |   ResourceSpec.cpu_cores_per_process
+          # after               |       X
+          # afterok             |       X
+          # afternotok          |       X
+          # afterany            |       X
+          # OOD does not have following PSI/J's interfaces.
+          #   JobSpec class:
+          #     pre_launch, post_launch, launcher
+          #   ResourceSpec class:
+          #     node_count, process_count, processes_per_node, exclusive_node_use
+
+          content = if script.shell_path.nil?
+            script.content
+          else
+            "#!#{script.shell_path}\n#{script.content}"
+          end
+
+          if ! script.native.nil?
+            native = script.native.join("\n") unless script.native.nil?
+            script.content.concat(native)
+          end
+
+          relative_path = "~/ood_tmp/run.sh"
+          full_path = File.expand_path("~/ood_tmp/run.sh")
+          FileUtils.mkdir_p(File.dirname(full_path))
+          File.open(full_path, "w") do |file|
+            file.write(content)
+          end
+
+          File.chmod(0755, full_path)
+
+          # convert OOD interfaces to PSI/J interfaces.
+          params = {
+            environment: script.job_environment,
+            directory: script.workdir,
+            name: script.job_name,
+            executable: relative_path,
+            stdin_path: script.input_path,
+            stdout_path: script.output_path,
+            stderr_path: script.error_path,
+            inherit_environment: script.copy_environment,
+            attributes: {queue_name: script.queue_name,
+                         reservation_id: script.reservation_id,
+                         account: script.accounting_id,
+                         duration: script.wall_time,
+                         custom_attributes: script.args},
+            resources: {__version: 1,
+                        gpu_cores_per_process: script.gpus_per_node,
+                        cpu_cores_per_process: script.cores}
+          }
+
+          if params[:attributes][:queue_name].nil?
+            params[:attributes][:queue_name] = @psij.queue_name
+          end
+          if params[:stdout_path].nil?
+            params[:stdout_path] = File.join(Dir.pwd, "stdout.txt")
+          end
+          if params[:stderr_path].nil?
+            params[:stderr_path] = File.join(Dir.pwd, "stderr.txt")
+          end
+
+          # add script.native to params[:attributes][:custom_attributes] of PSI/J.
+          if script.native && !script.native.empty?
+            if params[:attributes][:custom_attributes].nil?
+              params[:attributes][:custom_attributes] = script.native
+            else
+              params[:attributes][:custom_attributes].concat(script.native)
+            end
+          end
+          # Add script.native to params[:attributes][:cutsom_attributes] of PSI/J.
+          # Convert script.native array to hash.
+          # ['--<name>', 'value'] -> {name: value}
+          # ['--<name1>', '--<name2>'] -> {name1: "", name2: ""}
+          if ! params[:attributes][:custom_attributes].nil? 
+            hash = {}
+            skip = false
+            len = params[:attributes][:custom_attributes].length()-1
+            for index in 0..len do
+              if skip
+                skip = false
+                next
+              end
+              v = params[:attributes][:custom_attributes][index]
+              has_hyphen = false
+              if v.start_with?("--")
+                name = v[2..-1]
+                has_hyphen = true
+              elsif v.start_with?("-")
+                name = v[1..-1]
+                has_hyphen = true
+              else   
+                name = v
+              end
+              if index == len || !has_hyphen || params[:attributes][:custom_attributes][index+1].start_with?("-")
+                # if next value is not exist or start with "-", set empty string
+                hash[name] = ""
+              else
+                # if next value is exist and not start with "-", set value
+                hash[name] = params[:attributes][:custom_attributes][index+1]
+                skip = true
+              end
+            end
+            params[:attributes][:custom_attributes] = hash
+          end
+
+          # reject key which has nil value.
+          params[:attributes] = params[:attributes].reject {|_, value |value.nil?}
+          params[:resources] = params[:resources].reject {|_, value |value.nil?}
+          data = params.reject {|_, value |value.nil?}
+          
+          # serialize params to JSON
+          args = []
+          args[0] = @psij.executor
+
+          @psij.submit_job_path(args: args, chdir: script.workdir, stdin: JSON.generate(data))
+        rescue Batch::Error => e
+          raise JobAdapterError, e
+        end
+
+        def cluster_info
+        end
+
+        def accounts
+        end
+        
+        def delete(id)
+          id = id.to_s.strip()
+          params = {
+            id: id,
+            executor: @psij.executor,
+          }
+          args = params.map { |k, v| "--#{k}=#{v}" }
+          @psij.delete_job(args: args)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
+        end
+
+        def hold(id)
+          id = id.to_s.strip()
+          params = {
+            id: id,
+            executor: @psij.executor,
+          }
+          args = params.map { |k, v| "--#{k}=#{v}" }
+          @psij.hold_job(args: args)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
+        end
+
+        def release(id)
+          id = id.to_s.strip()
+          params = {
+            id: id,
+            executor: @psij.executor,
+          }
+          args = params.map { |k, v| "--#{k}=#{v}" }
+          @psij.release_job(args: args)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message unless /Invalid job id specified/ =~ e.message
+        end
+
+
+        def info(id)
+          id = id.to_s
+
+          job_infos = @psij.get_jobs(id: id).map do |v|
+            parse_job_info(v)
+          end
+
+          if job_infos.empty?
+            Info.new(id: id, status: :completed)
+          else
+            job_infos.first
+          end
+        rescue Batch::Error => e
+          # set completed status if can't find job id
+          if /Invalid job id specified/ =~ e.message
+            Info.new(
+              id: id,
+              status: :completed
+            )
+          else
+            raise JobAdapterError, e.message
+          end
+        end
+
+        def info_all(attrs: nil)
+          @psij.get_jobs.map do |v|
+            parse_job_info(v)
+          end
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        def info_where_owner(owner, attrs: nil)
+          owner = Array.wrap(owner).map(&:to_s).join(',')
+          @psij.get_jobs(owner: owner).map do |v|
+            parse_job_info(v)
+          end
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        def status(id)
+          info(id.to_s).status
+        end
+
+        def directive_prefix
+        end
+
+        private
+          def get_state(st)
+            STATE_MAP.fetch(st, :undetermined)
+          end
+
+          def parse_job_info(v)
+            # parse input hash to Info object
+            # if v don't have :reosurcelist, set empty array
+            if v[:resourcelist].nil? || v[:resourcelist].empty?
+              allocated_nodes = [ { name: "" } ]
+            else
+              allocated_nodes = v[:resourcelist]
+            end
+            if v[:cpu_time].nil?
+              cpu_time = nil
+            else
+              cpu_time = v[:cpu_time].to_i
+            end
+            Info.new(
+              id: v[:native_id],
+              status: get_state(v[:current_state]),
+              allocated_nodes: allocated_nodes,
+              submit_host: v[:submit_host],
+              job_name: v[:name],
+              job_owner: v[:owner],
+              accounting_id: v[:account],
+              procs: v[:process_count] ? v[:process_count].to_i : 0,
+              queue_name: v[:queue_name],
+              wallclock_time: v[:wall_time],
+              wallclock_limit: v[:duration],
+              cpu_time: cpu_time,
+              submission_time: v[:submission_time] ? Time.parse(v[:submission_time]): nil,
+              dispatch_time: v[:dispatch_time] ? Time.parse(v[:dispatch_time]): nil,
+              native: v
+            )
+          end
+
+      end
+    end
+  end
+end

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.28.0"
+  VERSION = "0.29.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.28.0
+  version: 0.29.0
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-07-16 00:00:00.000000000 Z
+date: 2025-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -210,6 +210,7 @@ files:
 - lib/ood_core/job/adapters/drmaa.rb
 - lib/ood_core/job/adapters/fujitsu_tcs.rb
 - lib/ood_core/job/adapters/helper.rb
+- lib/ood_core/job/adapters/htcondor.rb
 - lib/ood_core/job/adapters/kubernetes.rb
 - lib/ood_core/job/adapters/kubernetes/batch.rb
 - lib/ood_core/job/adapters/kubernetes/helper.rb
@@ -224,6 +225,12 @@ files:
 - lib/ood_core/job/adapters/lsf/batch.rb
 - lib/ood_core/job/adapters/lsf/helper.rb
 - lib/ood_core/job/adapters/pbspro.rb
+- lib/ood_core/job/adapters/psij.rb
+- lib/ood_core/job/adapters/psij/delete.py
+- lib/ood_core/job/adapters/psij/get_info.py
+- lib/ood_core/job/adapters/psij/hold.py
+- lib/ood_core/job/adapters/psij/release.py
+- lib/ood_core/job/adapters/psij/submit.py
 - lib/ood_core/job/adapters/sge.rb
 - lib/ood_core/job/adapters/sge/batch.rb
 - lib/ood_core/job/adapters/sge/helper.rb