resque-kubernetes 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f5cf6f4a4069b9440bbd5a136a24080214670c2
-  data.tar.gz: fac4fd9e289b57cce831e006459828481128c542
+  metadata.gz: e0881cb0e587e677806dbf25580ac7c22de17d29
+  data.tar.gz: 57a3680a1a064f9ac0d16b11c29c0bc5948862af
 SHA512:
-  metadata.gz: c36a6aafdb792d2b1cc1e3e137d6cc18a2f9cb0f3f0e8f1fc2ae9ee56489ea5688b1a9c4148fb92ea9d6accf0ccc9fd24ede2fd3ae132220a5ecbb7580ebdeb2
-  data.tar.gz: 365a4a5f2abeab6b8ad6ff2a4c032586fc19ce188f1ce04b2540df678d9a9d216e0909abe5e08c234cf6001ebc550599fe3386fa55f57269a83a130d32197cf5
+  metadata.gz: a73a284922f6e304bea0a3da941586e96dec3edb0d035e991f7a16e52f3ca80c5c8dc11ebc9aebba3879f213f0680e3d5ca34885cb3f6b1677b22a7859a6e41d
+  data.tar.gz: 5aded9eb8bfaabee8568e0f8ca8f1e5632c7ebfe65c8cdca2ade39a22bdf0cc3468a733a65938c5b79aceb6763e01decbe772ae3d58c50ca667864b6cd5d125c

data/.rubocop.yml

@@ -45,4 +45,4 @@ Lint/UselessSetterCall:
   Exclude:
     # Rubocop is incorrectly flagging `term_on_empty` as local
     # See: https://github.com/bbatsov/rubocop/issues/5420
-    - lib/resque/kubernetes/job.rb
+    - lib/resque/kubernetes/jobs_manager.rb

data/CHANGELOG.md

@@ -1,26 +1,32 @@
-# 0.8.0
+# v0.9.0
+- Update to not pollute the job class with our methods
+
+# v0.8.0
 - Fix bug where enqueueing jobs would keep adding workers if worker count
   was _greater_ than `max_workers`
-# 0.7.0
+
+# v0.7.0
 - Update to support kubeclient 2.2 or 3.x
-# 0.6.0
+
+# v0.6.0
 - Add support for ActiveJob when configured to be backed by Resque
 - When authorizing with `~/.kube/config` use Google Default Application Credentials rather than require a
   forked version of `kubeclient`
-# 0.5.0
+
+# v0.5.0
 - Maximum workers can no be configured per job type
 - Fix a crash when cleaning up a job that was removed by another process
 - No longer clean up pods because cleaning up finished jobs takes care of that
 - Apply rubocop, and bundler-audit rules
 
-# 0.4.0
+# v0.4.0
 - Syntax error fix
 
-# 0.3.0
+# v0.3.0
 - Syntax error fix
 
-# 0.2.0
+# v0.2.0
 - Fix for running in GKE cluster and production Rails environment
 
-# 0.1.0
+# v0.1.0
 - Initial release

data/lib/resque/kubernetes/job.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "kubeclient"
-
 module Resque
   module Kubernetes
     # Resque hook to autoscale Kubernetes Jobs for workers.
@@ -89,13 +87,12 @@ module Resque
           return unless Resque::Kubernetes.environments.include?(Rails.env)
         end
 
-        reap_finished_jobs
-        apply_kubernetes_job
+        manager = JobsManager.new(self)
+        manager.reap_finished_jobs
+        manager.apply_kubernetes_job
       end
 
-      protected
-
-      # Return the maximum number of workers to autoscale the job to.
+      # The maximum number of workers to autoscale the job to.
       #
       # While the number of active Kubernetes Jobs is less than this number,
       # the gem will add new Jobs to auto-scale the workers.
@@ -118,105 +115,6 @@ module Resque
       def max_workers
         Resque::Kubernetes.max_workers
       end
-
-      private
-
-      def jobs_client
-        return @jobs_client if @jobs_client
-        @jobs_client = client("/apis/batch")
-      end
-
-      def client(scope)
-        context = ContextFactory.context
-        return unless context
-
-        Kubeclient::Client.new(
-            context.api_endpoint + scope,
-            context.api_version,
-            ssl_options:  context.ssl_options,
-            auth_options: context.auth_options
-        )
-      end
-
-      def finished_jobs
-        resque_jobs = jobs_client.get_jobs(label_selector: "resque-kubernetes=job")
-        resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
-      end
-
-      def reap_finished_jobs
-        finished_jobs.each do |job|
-          begin
-            jobs_client.delete_job(job.metadata.name, job.metadata.namespace)
-          rescue KubeException => e
-            raise unless e.error_code == 404
-          end
-        end
-      end
-
-      def apply_kubernetes_job
-        manifest = DeepHash.new.merge!(job_manifest)
-        ensure_namespace(manifest)
-
-        # Do not start job if we have reached our maximum count
-        return if jobs_maxed?(manifest["metadata"]["name"], manifest["metadata"]["namespace"])
-
-        adjust_manifest(manifest)
-
-        job = Kubeclient::Resource.new(manifest)
-        jobs_client.create_job(job)
-      end
-
-      def jobs_maxed?(name, namespace)
-        resque_jobs = jobs_client.get_jobs(
-            label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
-            namespace:      namespace
-        )
-        running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
-        running.size >= max_workers
-      end
-
-      def adjust_manifest(manifest)
-        add_labels(manifest)
-        ensure_term_on_empty(manifest)
-        ensure_reset_policy(manifest)
-        update_job_name(manifest)
-      end
-
-      def add_labels(manifest)
-        manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
-        manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
-        manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
-      end
-
-      def ensure_term_on_empty(manifest)
-        manifest["spec"]["template"]["spec"] ||= {}
-        manifest["spec"]["template"]["spec"]["containers"] ||= []
-        manifest["spec"]["template"]["spec"]["containers"].each do |container|
-          container_term_on_empty(container)
-        end
-      end
-
-      def container_term_on_empty(container)
-        container["env"] ||= []
-        term_on_empty = container["env"].find { |env| env["name"] == "TERM_ON_EMPTY" }
-        unless term_on_empty
-          term_on_empty = {"name" => "TERM_ON_EMPTY"}
-          container["env"] << term_on_empty
-        end
-        term_on_empty["value"] = "1"
-      end
-
-      def ensure_reset_policy(manifest)
-        manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
-      end
-
-      def ensure_namespace(manifest)
-        manifest["metadata"]["namespace"] ||= "default"
-      end
-
-      def update_job_name(manifest)
-        manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
-      end
     end
   end
 end

data/lib/resque/kubernetes/jobs_manager.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "kubeclient"
+
+module Resque
+  module Kubernetes
+    # Spins up Kubernetes Jobs to run Resque workers.
+    class JobsManager
+      attr_reader :owner
+      private :owner
+
+      def initialize(owner)
+        @owner = owner
+      end
+
+      def reap_finished_jobs
+        finished_jobs.each do |job|
+          begin
+            jobs_client.delete_job(job.metadata.name, job.metadata.namespace)
+          rescue KubeException => e
+            raise unless e.error_code == 404
+          end
+        end
+      end
+
+      def apply_kubernetes_job
+        manifest = DeepHash.new.merge!(owner.job_manifest)
+        ensure_namespace(manifest)
+
+        # Do not start job if we have reached our maximum count
+        return if jobs_maxed?(manifest["metadata"]["name"], manifest["metadata"]["namespace"])
+
+        adjust_manifest(manifest)
+
+        job = Kubeclient::Resource.new(manifest)
+        jobs_client.create_job(job)
+      end
+
+      private
+
+      def jobs_client
+        return @jobs_client if @jobs_client
+        @jobs_client = client("/apis/batch")
+      end
+
+      def client(scope)
+        context = ContextFactory.context
+        return unless context
+
+        Kubeclient::Client.new(
+            context.api_endpoint + scope,
+            context.api_version,
+            ssl_options:  context.ssl_options,
+            auth_options: context.auth_options
+        )
+      end
+
+      def finished_jobs
+        resque_jobs = jobs_client.get_jobs(label_selector: "resque-kubernetes=job")
+        resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
+      end
+
+      def jobs_maxed?(name, namespace)
+        resque_jobs = jobs_client.get_jobs(
+            label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
+            namespace:      namespace
+        )
+        running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
+        running.size >= owner.max_workers
+      end
+
+      def adjust_manifest(manifest)
+        add_labels(manifest)
+        ensure_term_on_empty(manifest)
+        ensure_reset_policy(manifest)
+        update_job_name(manifest)
+      end
+
+      def add_labels(manifest)
+        manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
+        manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
+        manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
+      end
+
+      def ensure_term_on_empty(manifest)
+        manifest["spec"]["template"]["spec"] ||= {}
+        manifest["spec"]["template"]["spec"]["containers"] ||= []
+        manifest["spec"]["template"]["spec"]["containers"].each do |container|
+          container_term_on_empty(container)
+        end
+      end
+
+      def container_term_on_empty(container)
+        container["env"] ||= []
+        term_on_empty = container["env"].find { |env| env["name"] == "TERM_ON_EMPTY" }
+        unless term_on_empty
+          term_on_empty = {"name" => "TERM_ON_EMPTY"}
+          container["env"] << term_on_empty
+        end
+        term_on_empty["value"] = "1"
+      end
+
+      def ensure_reset_policy(manifest)
+        manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
+      end
+
+      def ensure_namespace(manifest)
+        manifest["metadata"]["namespace"] ||= "default"
+      end
+
+      def update_job_name(manifest)
+        manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
+      end
+    end
+  end
+end

data/lib/resque/kubernetes/version.rb

@@ -2,6 +2,6 @@
 
 module Resque
   module Kubernetes
-    VERSION = "0.8.0"
+    VERSION = "0.9.0"
   end
 end

data/lib/resque/kubernetes.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
+require "resque/kubernetes/configurable"
 require "resque/kubernetes/context_factory"
 require "resque/kubernetes/deep_hash"
 require "resque/kubernetes/dns_safe_random"
 require "resque/kubernetes/job"
+require "resque/kubernetes/jobs_manager"
 require "resque/kubernetes/version"
 require "resque/kubernetes/worker"
-require "resque/kubernetes/configurable"
 
 module Resque
   # Run Resque Jobs as Kubernetes Jobs with autoscaling.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: resque-kubernetes
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Jeremy Wadsack
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-05-22 00:00:00.000000000 Z
+date: 2018-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -160,6 +160,7 @@ files:
 - lib/resque/kubernetes/deep_hash.rb
 - lib/resque/kubernetes/dns_safe_random.rb
 - lib/resque/kubernetes/job.rb
+- lib/resque/kubernetes/jobs_manager.rb
 - lib/resque/kubernetes/version.rb
 - lib/resque/kubernetes/worker.rb
 - resque-kubernetes.gemspec