resque-kubernetes 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd653a29a38d097f457584f8bb8a4b305dc14f1f
-  data.tar.gz: 4ac673bc617fdd91c99fe3b51f0c943f214655f6
+  metadata.gz: bbfbc8c610d079165438b6a635043070730ecc0a
+  data.tar.gz: a409a4638079687074703bbe70e0acb7ab92d40f
 SHA512:
-  metadata.gz: 142a38eea17c227b6339eef837b58e4835d3fdf05d7f76f85f30cde00c25f6d20b3ee1fa59795b030549283bfcae28eb49da8159449bd3004066c258c5a7b4d0
-  data.tar.gz: 707c7a76de380e87d8da7c4f4651d924f9b7992d5bdedcf48150a0ce8c225cda21d9f252f4a4b907dae373c5c012be2f9ec449bb69d90501a711607f2161b3e2
+  metadata.gz: 03e42650b7af55aca5d0eacb803cc58b30f70ebc4148dc8226e39242c266b061824d1a38f3291b5e5b2a8961d216703d5d03fd2ea31faef4027ddc4b1e097d81
+  data.tar.gz: 849ebc1a05f62eb3676d7dfc6550b975f15da62172cdaba86339ad3c8974cf9610c889f72084de908d8739598ff5447b1f5dcaf1e595ede9af6c32c0802685a2

data/.rubocop.yml

@@ -31,6 +31,9 @@ Style/FormatStringToken:
   EnforcedStyle: template
 Style/AsciiComments:
   Enabled: false
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - "gemfiles/*"
 
 Metrics/BlockLength:
   Exclude:
@@ -45,4 +48,4 @@ Lint/UselessSetterCall:
   Exclude:
     # Rubocop is incorrectly flagging `term_on_empty` as local
     # See: https://github.com/bbatsov/rubocop/issues/5420
-    - lib/resque/kubernetes/jobs_manager.rb
+    - lib/resque/kubernetes/manifest_conformance.rb

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# v1.2.0
+- Clean up finished pods that have successfully completed
+
 # v1.1.0
 - Fix design to set `INTERVAL=0` for the worker in the Kubernetes
   job manifest, which tells `resque` to work until there are no more jobs,

data/README.md

@@ -1,4 +1,5 @@
 # Resque::Kubernetes
+[![Gem Version](https://badge.fury.io/rb/resque-kubernetes.svg)](https://badge.fury.io/rb/resque-kubernetes)
 
 Run Resque (and ActiveJob) Workers as Kubernetes Jobs!
 

data/lib/resque/kubernetes.rb

@@ -8,6 +8,7 @@ require "resque/kubernetes/deep_hash"
 require "resque/kubernetes/dns_safe_random"
 require "resque/kubernetes/job"
 require "resque/kubernetes/jobs_manager"
+require "resque/kubernetes/manifest_conformance"
 require "resque/kubernetes/version"
 
 module Resque

data/lib/resque/kubernetes/dns_safe_random.rb

@@ -12,13 +12,13 @@ module Resque
       class << self
         # Returns an n-length string of DNS-safe characters.
         #
-        # n: The number of characters to return (default 5).
-        def random_chars(n = 5)
-          s = [SecureRandom.random_bytes(n)].pack("m*")
+        # number: The number of characters to return (default 5).
+        def random_chars(number = 5)
+          s = [SecureRandom.random_bytes(number)].pack("m*")
           s.delete!("=\n")
           s.tr!("+/_-", "0")
           s.tr!("A-Z", "a-z")
-          s[0...n]
+          s[0...number]
         end
       end
     end

data/lib/resque/kubernetes/job.rb

@@ -82,13 +82,14 @@ module Resque
       end
 
       # A before_enqueue hook that adds worker jobs to the cluster.
-      def before_enqueue_kubernetes_job(*_)
+      def before_enqueue_kubernetes_job(*_args)
         if defined? Rails
           return unless Resque::Kubernetes.environments.include?(Rails.env)
         end
 
         manager = JobsManager.new(self)
         manager.reap_finished_jobs
+        manager.reap_finished_pods
         manager.apply_kubernetes_job
       end
 

data/lib/resque/kubernetes/jobs_manager.rb

@@ -2,10 +2,14 @@
 
 require "kubeclient"
 
+require_relative "manifest_conformance"
+
 module Resque
   module Kubernetes
     # Spins up Kubernetes Jobs to run Resque workers.
     class JobsManager
+      include Resque::Kubernetes::ManifestConformance
+
       attr_reader :owner
       private :owner
 
@@ -24,6 +28,16 @@ module Resque
         end
       end
 
+      def reap_finished_pods
+        finished_pods.each do |pod|
+          begin
+            pods_client.delete_pod(pod.metadata.name, pod.metadata.namespace)
+          rescue KubeException => e
+            raise unless e.error_code == 404
+          end
+        end
+      end
+
       def apply_kubernetes_job
         manifest = DeepHash.new.merge!(owner.job_manifest)
         ensure_namespace(manifest)
@@ -43,6 +57,10 @@ module Resque
         @jobs_client ||= client("/apis/batch")
       end
 
+      def pods_client
+        @pods_client ||= client("")
+      end
+
       def client(scope)
         return Resque::Kubernetes.kubeclient if Resque::Kubernetes.kubeclient
 
@@ -58,6 +76,15 @@ module Resque
         resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
       end
 
+      def finished_pods
+        resque_jobs = pods_client.get_pods(label_selector: "resque-kubernetes=pod")
+        resque_jobs.select do |pod|
+          pod.status.phase == "Succeeded" && pod.status.containerStatuses.all? do |status|
+            status.state.terminated.reason == "Completed"
+          end
+        end
+      end
+
       def jobs_maxed?(name, namespace)
         resque_jobs = jobs_client.get_jobs(
             label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
@@ -66,49 +93,6 @@ module Resque
         running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
         running.size >= owner.max_workers
       end
-
-      def adjust_manifest(manifest)
-        add_labels(manifest)
-        ensure_term_on_empty(manifest)
-        ensure_reset_policy(manifest)
-        update_job_name(manifest)
-      end
-
-      def add_labels(manifest)
-        manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
-        manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
-        manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
-      end
-
-      def ensure_term_on_empty(manifest)
-        manifest["spec"]["template"]["spec"] ||= {}
-        manifest["spec"]["template"]["spec"]["containers"] ||= []
-        manifest["spec"]["template"]["spec"]["containers"].each do |container|
-          container_term_on_empty(container)
-        end
-      end
-
-      def container_term_on_empty(container)
-        container["env"] ||= []
-        term_on_empty = container["env"].find { |env| env["name"] == "INTERVAL" }
-        unless term_on_empty
-          term_on_empty = {"name" => "INTERVAL"}
-          container["env"] << term_on_empty
-        end
-        term_on_empty["value"] = "0"
-      end
-
-      def ensure_reset_policy(manifest)
-        manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
-      end
-
-      def ensure_namespace(manifest)
-        manifest["metadata"]["namespace"] ||= @default_namespace
-      end
-
-      def update_job_name(manifest)
-        manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
-      end
     end
   end
 end

data/lib/resque/kubernetes/manifest_conformance.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Resque
+  module Kubernetes
+    # Provides methods to ensure a mannifest conforms to a job specification
+    # and includes details needed for resque-kubernetes
+    module ManifestConformance
+      def adjust_manifest(manifest)
+        add_labels(manifest)
+        ensure_term_on_empty(manifest)
+        ensure_reset_policy(manifest)
+        update_job_name(manifest)
+      end
+
+      def add_labels(manifest)
+        manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
+        manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
+        manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
+      end
+
+      def ensure_term_on_empty(manifest)
+        manifest["spec"]["template"]["spec"] ||= {}
+        manifest["spec"]["template"]["spec"]["containers"] ||= []
+        manifest["spec"]["template"]["spec"]["containers"].each do |container|
+          container_term_on_empty(container)
+        end
+      end
+
+      def container_term_on_empty(container)
+        container["env"] ||= []
+        term_on_empty = container["env"].find { |env| env["name"] == "INTERVAL" }
+        unless term_on_empty
+          term_on_empty = {"name" => "INTERVAL"}
+          container["env"] << term_on_empty
+        end
+        term_on_empty["value"] = "0"
+      end
+
+      def ensure_reset_policy(manifest)
+        manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
+      end
+
+      def ensure_namespace(manifest)
+        manifest["metadata"]["namespace"] ||= @default_namespace
+      end
+
+      def update_job_name(manifest)
+        manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
+      end
+    end
+  end
+end

data/lib/resque/kubernetes/version.rb

@@ -2,6 +2,6 @@
 
 module Resque
   module Kubernetes
-    VERSION = "1.1.0"
+    VERSION = "1.2.0"
   end
 end

data/resque-kubernetes.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path("lib", __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "resque/kubernetes/version"
 
@@ -29,7 +29,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "googleauth", "~> 0.6"
   spec.add_development_dependency "rake", "~> 12.3"
   spec.add_development_dependency "rspec", "~> 3.7"
-  spec.add_development_dependency "rubocop", "~> 0.52", ">= 0.52.1"
+  # Pinned less than 0.56.0 until this is resolved: https://github.com/rubocop-hq/rubocop/issues/5975
+  spec.add_development_dependency "rubocop", "~> 0.52", ">= 0.52.1", "< 0.56.0"
 
   spec.add_dependency "kubeclient", ">= 3.1.2", "< 5.0"
   spec.add_dependency "resque", "~> 1.26"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: resque-kubernetes
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Jeremy Wadsack
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-27 00:00:00.000000000 Z
+date: 2018-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: appraisal
@@ -104,6 +104,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.52.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.56.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -114,6 +117,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.52.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.56.0
 - !ruby/object:Gem::Dependency
   name: kubeclient
   requirement: !ruby/object:Gem::Requirement
@@ -180,6 +186,7 @@ files:
 - lib/resque/kubernetes/dns_safe_random.rb
 - lib/resque/kubernetes/job.rb
 - lib/resque/kubernetes/jobs_manager.rb
+- lib/resque/kubernetes/manifest_conformance.rb
 - lib/resque/kubernetes/version.rb
 - resque-kubernetes.gemspec
 homepage: https://github.com/keylimetoolbox/resque-kubernetes