ood_core 0.17.6 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5611442cd9ae16b3716ae1b7acc1188868590447745ef8cebd74a009eebf34b
-  data.tar.gz: 07226bc4c9dcdf6bd82a8adce1b738adbb9730ec4a64cdbed2313f8a7724c0c2
+  metadata.gz: 8564931d7e160d2b23c383d60c8001de98a1a211cd7f0418a442dd99b0c48f78
+  data.tar.gz: af7585fe95630e9e6aa04a004c0650ad671d36894bf1d82554bf75e87056faca
 SHA512:
-  metadata.gz: 6ba4ced751b6ee01c55ef9a2ceb9efeafe317b6f5b2a351ba231a23a4a1999ff00c62a9ba595e83b1101216fdd89f2d9b18c6d34ed7684af05a75fef87d673d5
-  data.tar.gz: 3ff4e64fac908455303e2df27d1a1825f1ebbf234caa0d8eb1be6a5a5b0b89122d6fb2c4490139c49b4022b088e85ddeb73104ef6b1b6be1e66771f1921aeb11
+  metadata.gz: bd40c9483ee3c7a3bf036c9442c7e0e5f96d819b99c63248252050ecceb7a933f8636d1e55176c3271db7503ecc65a7c29f5d0add3f4dfe85accf1a35e25949e
+  data.tar.gz: 3c8a28b6623b388ea75a5bc418a5d6fcb1738ee2c45f0ae78b0a54d56305ed19ccce26e9b183674633c0c67d03e93fff002438e3c750c674fe4781e948d58d90

data/CHANGELOG.md

@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [0.18.0] - 10-18-2021
+
+### Fixed
+
+- Fixed LHA crashing on strange bash output in [322](https://github.com/OSC/ood_core/pull/322).
+
+### Added
+
+- All adapters now respond to #{adapter}? methods like slurm?, pbspro?, kubernetes? and so on
+  in [326](https://github.com/OSC/ood_core/pull/326).
+
+### Changed
+
+- The kubernetes adapter now expects to set context statically in [324](https://github.com/OSC/ood_core/pull/324).
+  And can now accept context as a part of it's interface. It will now also always send --context when using OIDC
+  and that context defaults to the clustername in [327](https://github.com/OSC/ood_core/pull/327).
+- Removed the activesupport dependency in [329](https://github.com/OSC/ood_core/pull/329).
+
 ## [0.17.6] - 8-24-2021
 
 ### Added
@@ -387,7 +405,8 @@ Functionally the same as [0.17.3] but with some CI updates.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.17.6...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.18.0...HEAD
+[0.18.0]: https://github.com/OSC/ood_core/compare/v0.17.8...v0.18.0
 [0.17.6]: https://github.com/OSC/ood_core/compare/v0.17.5...v0.17.6
 [0.17.5]: https://github.com/OSC/ood_core/compare/v0.17.4...v0.17.5
 [0.17.4]: https://github.com/OSC/ood_core/compare/v0.17.3...v0.17.4

data/RAILS-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2005-2021 David Heinemeier Hansson
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/ood_core/cluster.rb

@@ -63,6 +63,20 @@ module OodCore
       @errors          = c.fetch(:errors, [])  .to_a
     end
 
+
+    # programatically define some methods like slurm? or torque?
+    Dir.entries("#{__dir__}/job/adapters").select do |f|
+      File.file?("#{__dir__}/job/adapters/#{f}") && File.extname(f) == '.rb'
+    end.map do |f|
+      File.basename(f, '.rb')
+    end.reject do |f|
+      ['helper', 'drmaa' ].include?(f)
+    end.each do |adapter|
+      define_method(:"#{adapter}?")  do
+        job_config.fetch(:adapter, nil) == adapter
+      end
+    end
+
     # Metadata that provides extra information about this cluster
     # @return [OpenStruct] the metadata
     def metadata

data/lib/ood_core/job/adapters/kubernetes/batch.rb

@@ -11,15 +11,15 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   class Error < StandardError; end
   class NotFoundError < StandardError; end
 
-  attr_reader :config_file, :bin, :cluster, :mounts
-  attr_reader :all_namespaces, :using_context, :helper
+  attr_reader :config_file, :bin, :cluster, :context, :mounts
+  attr_reader :all_namespaces, :helper
   attr_reader :username_prefix, :namespace_prefix
   attr_reader :auto_supplemental_groups
 
   def initialize(options = {})
     options = options.to_h.symbolize_keys
 
-    @config_file = options.fetch(:config_file, default_config_file)
+    @config_file = options.fetch(:config_file, Batch.default_config_file)
     @bin = options.fetch(:bin, '/usr/bin/kubectl')
     @cluster = options.fetch(:cluster, 'open-ondemand')
     @mounts = options.fetch(:mounts, []).map { |m| m.to_h.symbolize_keys }
@@ -28,15 +28,10 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     @namespace_prefix = options.fetch(:namespace_prefix, '')
     @auto_supplemental_groups = options.fetch(:auto_supplemental_groups, false)
 
-    @using_context = false
-    @helper = OodCore::Job::Adapters::Kubernetes::Helper.new
+    tmp_ctx = options.fetch(:context, nil)
+    @context = tmp_ctx.nil? && oidc_auth?(options.fetch(:auth, {})) ? @cluster : tmp_ctx
 
-    begin
-      make_kubectl_config(options)
-    rescue
-      # FIXME could use a log here
-      # means you couldn't 'kubectl set config'
-    end
+    @helper = OodCore::Job::Adapters::Kubernetes::Helper.new
   end
 
   def resource_file(resource_type = 'pod')
@@ -61,10 +56,10 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   end
 
   def info_all(attrs: nil)
-    cmd = if all_namespaces
-            "#{base_cmd} get pods -o json --all-namespaces"
+    cmd = if @all_namespaces
+            "#{base_cmd} -o json get pods --all-namespaces"
           else
-            "#{namespaced_cmd} get pods -o json"
+            "#{namespaced_cmd} -o json get pods"
           end
 
     output = call(cmd)
@@ -117,6 +112,32 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     safe_call("delete", "configmap", configmap_name(id))
   end
 
+  class << self
+    def default_config_file
+      (ENV['KUBECONFIG'] || "#{Dir.home}/.kube/config")
+    end
+
+    def default_auth
+      {
+        type: 'managed'
+      }.symbolize_keys
+    end
+
+    def default_server
+      {
+        endpoint: 'https://localhost:8080',
+        cert_authority_file: nil
+      }.symbolize_keys
+    end
+
+    def configure_kube!(config)
+      k = self.new(config)
+      # TODO: probably shouldn't be using send here
+      k.send(:set_cluster, config.fetch(:server, default_server).to_h.symbolize_keys)
+      k.send(:configure_auth, config.fetch(:auth, default_auth).to_h.symbolize_keys)
+    end
+  end
+
   private
 
   def safe_call(verb, resource, id)
@@ -227,6 +248,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   # and  id=my-pod-id
   def call_json_output(verb, resource, id, stdin: nil)
     cmd = "#{formatted_ns_cmd} #{verb} #{resource} #{id}"
+
     data = call(cmd, stdin: stdin)
     data = data.empty? ? '{}' : data
     json_data = JSON.parse(data, symbolize_names: true)
@@ -250,27 +272,6 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     "#{namespace_prefix}#{username}"
   end
 
-  def context
-    cluster
-  end
-
-  def default_config_file
-    (ENV['KUBECONFIG'] || "#{Dir.home}/.kube/config")
-  end
-
-  def default_auth
-    {
-      type: 'managaged'
-    }.symbolize_keys
-  end
-
-  def default_server
-    {
-      endpoint: 'https://localhost:8080',
-      cert_authority_file: nil
-    }.symbolize_keys
-  end
-
   def formatted_ns_cmd
     "#{namespaced_cmd} -o json"
   end
@@ -281,7 +282,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
 
   def base_cmd
     base = "#{bin} --kubeconfig=#{config_file}"
-    base << " --context=#{context}" if using_context
+    base << " --context=#{context}" if context?
     base
   end
 
@@ -309,28 +310,30 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     nil
   end
 
-  def make_kubectl_config(config)
-    set_cluster(config.fetch(:server, default_server).to_h.symbolize_keys)
-    configure_auth(config.fetch(:auth, default_auth).to_h.symbolize_keys)
-  end
-
   def configure_auth(auth)
-    type = auth.fetch(:type)
-    return if managed?(type)
-
-    case type
-    when 'gke'
+    if managed_auth?(auth)
+      return
+    elsif gke_auth?(auth)
       set_gke_config(auth)
-    when 'oidc'
+    elsif oidc_auth?(auth)
       set_context
     end
   end
 
-  def use_context
-    @using_context = true
+  def context?
+    !@context.nil?
+  end
+
+  def gke_auth?(auth = {})
+    auth.fetch(:type, nil) == 'gke'
+  end
+
+  def oidc_auth?(auth = {})
+    auth.fetch(:type, nil) == 'oidc'
   end
 
-  def managed?(type)
+  def managed_auth?(auth = {})
+    type = auth.fetch(:type, nil)
     if type.nil?
       true # maybe should be false?
     else
@@ -359,23 +362,24 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     # gke cluster name can probably can differ from what ood calls the cluster
     cmd = "gcloud container clusters get-credentials #{locale} #{cluster}"
     env = { 'KUBECONFIG' => config_file }
-    call(cmd, env)
+    call(cmd, env: env)
   end
 
   def set_context
-    cmd = "#{base_cmd} config set-context #{cluster}"
+    # can't really use base_cmd, bc it may use --context flag
+    cmd = "#{bin} --kubeconfig=#{config_file} config set-context #{context}"
     cmd << " --cluster=#{cluster} --namespace=#{namespace}"
     cmd << " --user=#{k8s_username}"
 
     call(cmd)
-    use_context
   end
 
   def set_cluster(config)
     server = config.fetch(:endpoint)
     cert = config.fetch(:cert_authority_file, nil)
 
-    cmd = "#{base_cmd} config set-cluster #{cluster}"
+    # shouldn't use context here either
+    cmd = "#{bin} --kubeconfig=#{config_file} config set-cluster #{cluster}"
     cmd << " --server=#{server}"
     cmd << " --certificate-authority=#{cert}" unless cert.nil?
 

data/lib/ood_core/job/adapters/kubernetes/helper.rb

@@ -4,7 +4,9 @@ class OodCore::Job::Adapters::Kubernetes::Helper
   require_relative 'k8s_job_info'
   require 'resolv'
   require 'base64'
-  require 'active_support/core_ext/hash'
+  require 'ood_core/refinements/hash_extensions'
+
+  using OodCore::Refinements::HashExtensions
 
   class K8sDataError < StandardError; end
 

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -61,7 +61,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
 
     session_name = unique_session_name
     output = call(*cmd, stdin: wrapped_script(script, session_name))
-    hostname = output.strip
+    hostname = parse_hostname(output)
 
     "#{session_name}@#{hostname}"
   end
@@ -242,22 +242,20 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
   def list_remote_tmux_session(destination_host)
     # Note that the tmux variable substitution looks like Ruby string sub,
     # these must either be single quoted strings or Ruby-string escaped as well
-    format_str = Shellwords.escape(
-      ['#{session_name}', '#{session_created}', '#{pane_pid}'].join(UNIT_SEPARATOR)
-    )
+    format_str = Shellwords.escape(['#{session_name}', '#{session_created}', '#{pane_pid}'].join(UNIT_SEPARATOR))
     keys = [:session_name, :session_created, :session_pid]
     cmd = ssh_cmd(destination_host, ['tmux', 'list-panes', '-aF', format_str])
-
-    call(*cmd).split(
-      "\n"
-    ).map do |line|
+    
+    call(*cmd).split("\n").map do |line|
       Hash[keys.zip(line.split(UNIT_SEPARATOR))].tap do |session_hash|
         session_hash[:destination_host] = destination_host
         session_hash[:id] = "#{session_hash[:session_name]}@#{destination_host}"
       end
-    end.select{
-      |session_hash| session_hash[:session_name].start_with?(session_name_label)
-    }
+    end.select do |session_hash| 
+      session_hash.compact.length >= 5 && 
+        !session_hash[:session_name].nil? && 
+        session_hash[:session_name].start_with?(session_name_label)
+    end
   rescue Error => e
     interpret_and_raise(e)
     []
@@ -287,4 +285,10 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
       raise error
     end
   end
+
+  def parse_hostname(output)
+    output.split($/).map do |line|
+      line.match(/^(([\w+]|[a-zA-Z0-9][\w*-]*\.))*$/)
+    end.compact.last.to_s
+  end
 end

data/lib/ood_core/job/adapters/linux_host/templates/script_wrapper.erb.sh

@@ -13,6 +13,7 @@ if [ -z "$hostname" ]; then
     exit 1
 fi
 
+echo ""
 echo $hostname
 
 # Put the script into a temp file on localhost

data/lib/ood_core/refinements/hash_extensions.rb

@@ -2,6 +2,8 @@ module OodCore
   # Namespace for Ruby refinements
   module Refinements
     # This module provides refinements for manipulating the Ruby {Hash} class.
+    # Some elements have been taken from Rails (https://github.com/rails/rails)
+    # and it's LICENSE has been added as RAILS-LICENSE in the root directory of this project.
     module HashExtensions
       refine Hash do
         # Symbolize the keys in a {Hash}
@@ -28,6 +30,37 @@ module OodCore
         def compact
           self.select { |_, value| !value.nil? }
         end
+
+        # Returns a new hash with +self+ and +other_hash+ merged recursively.
+        #
+        #   h1 = { a: true, b: { c: [1, 2, 3] } }
+        #   h2 = { a: false, b: { x: [3, 4, 5] } }
+        #
+        #   h1.deep_merge(h2) # => { a: false, b: { c: [1, 2, 3], x: [3, 4, 5] } }
+        #
+        # Like with Hash#merge in the standard library, a block can be provided
+        # to merge values:
+        #
+        #   h1 = { a: 100, b: 200, c: { c1: 100 } }
+        #   h2 = { b: 250, c: { c1: 200 } }
+        #   h1.deep_merge(h2) { |key, this_val, other_val| this_val + other_val }
+        #   # => { a: 100, b: 450, c: { c1: 300 } }
+        def deep_merge(other_hash, &block)
+          dup.deep_merge!(other_hash, &block)
+        end
+
+        # Same as +deep_merge+, but modifies +self+.
+        def deep_merge!(other_hash, &block)
+          merge!(other_hash) do |key, this_val, other_val|
+            if this_val.is_a?(Hash) && other_val.is_a?(Hash)
+              this_val.deep_merge(other_val, &block)
+            elsif block_given?
+              block.call(key, this_val, other_val)
+            else
+              other_val
+            end
+          end
+        end
       end
     end
   end

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.17.6"
+  VERSION = "0.18.0"
 end

data/ood_core.gemspec

@@ -25,7 +25,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "ood_support", "~> 0.0.2"
   spec.add_runtime_dependency "ffi", "~> 1.9", ">= 1.9.6"
   spec.add_development_dependency "bundler", "~> 2.1"
-  spec.add_runtime_dependency "activesupport", ">= 5.2", "< 6.0"
   spec.add_development_dependency "rake", "~> 13.0.1"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry", "~> 0.10"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.17.6
+  version: 0.18.0
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-08-24 00:00:00.000000000 Z
+date: 2021-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -60,26 +60,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
-- !ruby/object:Gem::Dependency
-  name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -165,6 +145,7 @@ files:
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
+- RAILS-LICENSE
 - README.md
 - Rakefile
 - bin/console