resque-kubernetes 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f22de9195620c7f00040e39027aebaefccb4e52
-  data.tar.gz: 524587419d578689b3719c16bf21105a26fdfa03
+  metadata.gz: acdff0738bfd9d4f3aeb8c2ba1f8eb744b5d3f59
+  data.tar.gz: 80bafaa288815eb1ddfdea7041e0edc302b5e7f0
 SHA512:
-  metadata.gz: 428eba2f1ba7707ccc9992a684530b50b83f280b824b297984b583ac068d3fdf738bcd8edd5806ac49865d9666fea41cd4f6a7a255790955d0c48c2d8a232113
-  data.tar.gz: e87c50ee0bd9742e5495707069bbf97dd90a7f080bf78a5865a39ef912f6f59889d5feddddce309f5469ebfe67c9955c9909561714fe940d71994f0f3c6ed869
+  metadata.gz: a542e73f31e4e9eff7194ef03e356185d4e0cd89820b3f59b043b5521c58b557493ec76c37d2cd6c838e640daa66581aff8a5e7a44d212886743210f943733f8
+  data.tar.gz: 46132bfd101f9353cdc0eddd3777825ee93f893f2d3ddb9c0c33f645a75b3a477898f6f77c9bff02bb62c02d0ecbb5c5fc615a9f6355c37acb5edbccfa148c68

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+*.gem

data/.rubocop.yml

@@ -34,7 +34,7 @@ Style/AsciiComments:
 
 Metrics/BlockLength:
   Exclude:
-    - "keylime-service-api.gemspec"
+    - "*.gemspec"
     - "spec/**/*"
 
 Layout/EmptyLinesAroundBlockBody:

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.6.0
+- Add support for ActiveJob when configured to be backed by Resque
+- When authorizing with `~/.kube/config` use Google Default Application Credentials rather than require a
+  forked version of `kubeclient`
 # 0.5.0
 - Maximum workers can no be configured per job type
 - Fix a crash when cleaning up a job that was removed by another process

data/README.md

@@ -1,28 +1,28 @@
 # Resque::Kubernetes
 
-Run Resque Jobs as Kubernetes Jobs!
+Run Resque (and ActiveJob) Workers as Kubernetes Jobs!
 
 Kubernetes has a concept of "Job" which is a pod that runs a container until
 the container finishes and then it terminates the pod (as opposed to trying to
 restart the container).
 
-This gem takes advantage of that feature by starting up a Kubernetes Job when 
-a Resque Job is enqueued. It then allows the Resque Worker to be modified to 
-terminate when there are no more jobs in the queue.
+This gem takes advantage of that feature by starting up a Kubernetes Job to
+run a worker when a Resque job or ActiveJob is enqueued. It then allows the
+Resque worker to be modified to terminate when there are no more jobs in the queue.
 
 Why would you do this?
 
 We have unpredictable, resource-intensive jobs. Rather than dedicating large 
 nodes in our cluster to run the resque workers, where the resources would be 
 idle when there are no jobs to run, we can use auto-scaling to add nodes when
-Kubernetes Job gets created and shut them down when those jobs are complete. 
+a Kubernetes Job gets created and shut them down when those jobs are complete. 
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'resque-kubernetes'
+gem "resque-kubernetes"
 ```
 
 And then execute:
@@ -35,42 +35,96 @@ Or install it yourself as:
 
 ## Usage
 
-For any Resque job that you want to run in a Kubernetes job, you'll need to
-modify the job class with two things:
+This works with native/pure Resque jobs and with ActiveJob _backed by Resque_.
+Under ActiveJob, the workers are still Resque workers, so the same set up 
+applies. You just configure the job class differently.
 
-- extend the class with `Resque::Kubernetes::Job`
-- and add a method `job_manifest` that returns the Kubernetes manifest for the job
+### Pure Resque
+
+For any Resque job that you want to run in a Kubernetes job, you'll need
+to modify the class with two things:
+
+- `extend` the class with `Resque::Kubernetes::Job`
+- add a class method `job_manifest` that returns the Kubernetes manifest for the job
+  as a `Hash`
 
 ```ruby
 class ResourceIntensiveJob
   extend Resque::Kubernetes::Job
-  
+
+  class << self
+    def perform
+      # ... your existing code
+    end
+
+    def job_manifest
+      YAML.safe_load(
+        <<~MANIFEST
+          apiVersion: batch/v1
+          kind: Job
+          metadata:
+            name: worker-job
+          spec:
+            template:
+              metadata:
+                name: worker-job
+              spec:
+                containers:
+                - name: worker
+                  image: us.gcr.io/project-id/some-resque-worker
+                  env:
+                  - name: QUEUE
+                    value: high-memory
+        MANIFEST
+      )
+    end
+  end
+end
+```
+
+### ActiveJob (on Resque)
+
+For any ActiveJob that you want to run in a Kubernetes job, you'll need to
+modify the class with two things:
+
+- `include` `Resque::Kubernetes::Job` in the class
+- add an instance method `job_manifest` that returns the Kubernetes manifest for the job
+  as a `Hash`
+
+```ruby
+class ResourceIntensiveJob < ApplicationJob
+  include Resque::Kubernetes::Job
+
   def perform
     # ... your existing code
   end
-  
+
   def job_manifest
-    <<-EOD
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: worker-job
-spec:
-  template:
-    metadata:
-      name: worker-job
-    spec:
-      containers:
-      - name: worker
-        image: us.gcr.io/project-id/some-resque-worker
-        env:
-        - name: QUEUE
-          value: high-memory
-    EOD
+    YAML.safe_load(
+      <<~MANIFEST
+        apiVersion: batch/v1
+        kind: Job
+        metadata:
+          name: worker-job
+        spec:
+          template:
+            metadata:
+              name: worker-job
+            spec:
+              containers:
+              - name: worker
+                image: us.gcr.io/project-id/some-resque-worker
+                env:
+                - name: QUEUE
+                  value: high-memory
+      MANIFEST
+    )
   end
 end
 ```
 
+### Workers (for both)
+
 Make sure that the container image above, which is used to run the resque 
 worker, is built to include the `resque-kubernetes` gem as well. The gem will 
 add `TERM_ON_EMPTY` to the environment variables. This tells the worker that 
@@ -78,6 +132,13 @@ whenever the queue is empty it should terminate the worker. Kubernetes will
 then terminate the Job when the container is done running and will release the 
 resources.
 
+### Job manifest
+
+In the example above we show the manifest as a HEREDOC, just to make it
+simple. But you could also read this from a file, parse a template and insert
+values, or anything else you want to do in the method, as long as you return
+a valid Kubernetes Job manifest as a `Hash`.
+
 ## Configuration
 
 You can modify the configuration of the gem by creating an initializer in
@@ -87,28 +148,26 @@ your project:
 # config/initializers/resque-kubernetes.rb
 
 Resque::Kubernetes.configuration do |config|
-
  config.environments << "staging"
  config.max_workers = 10
-
 end
 ```
 
 ### `environments`
 
+> This only works under Rails, when `Rails.env` is set.
+
 By default `Resque::Kubernetes` will only manage Kubernetes Jobs in
 `:production`. If you want to add other environments you can update this list
 (`config.environments << "staging"`) or replace it (`config.environments =
 ["production", "development"]`).
 
-Note that this only works under Rails, when `Rails.env` is set.
-
 ### `max_workers`
 
 `Resque::Kubernetes` will spin up a Kuberentes Job each time you enqueue a 
 Resque Job. This allows for parallel processing of jobs using the resources
-available to your cluster. By default this is limited to 10 workers, so an not
-to have run-away cloud resource usage.
+available to your cluster. By default this is limited to 10 workers, to prevent 
+run-away cloud resource usage.
 
 You can set this higher if you need massive scaling and your structure supports
 it.
@@ -116,7 +175,7 @@ it.
 If you don't want more than one job running at a time then set this to 1.
 
 Beyond this global scope you can adjust the total number of workers on each
-individual Resque Job type by overriding the `max_workers` class method for the job.
+individual Resque Job type by overriding the `max_workers` method for the job.
 If you change this, the value returned by that method takes precedence over the
 global value.
 
@@ -124,38 +183,43 @@ global value.
 class ResourceIntensiveJob
   extend Resque::Kubernetes::Job
 
-  def perform
-    # ...
-  end
+  class << self
+    def perform
+      # ...
+    end
 
-  def job_manifest
-    # ...
-  end
+    def job_manifest
+      # ...
+    end
 
-  def max_workers
-    # Simply return an integer value, or do something more complicated if needed.
-    105
+    def max_workers
+      # Simply return an integer value, or do something more complicated if needed.
+      105
+    end
   end
 end
 ```
 
 ## To Do
 
-- We probably need better namespace support, particularly for reaping 
-  finished jobs and pods.
 - Support for other authentication and server URL options for `kubeclient`.
   See [the many examples](https://github.com/abonas/kubeclient#usage) in their
   README.
+- We probably need better namespace support, particularly for reaping
+  finished jobs and pods.
 
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, 
-run `rake spec` to run the tests. You can also run `bin/console` for an 
-interactive prompt that will allow you to experiment.
+run `rake spec` to run the tests.
+
+You can run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
 
-To install this gem onto your local machine, run `bundle exec rake install`. 
-To release a new version, update the version number in `version.rb`, and then 
-run `bundle exec rake release`, which will create a git tag for the version, 
+To release a new version, update the version number in
+`lib/resque/kubernetes/version.rb` and the `CHANGELOG.md`, then run
+`bundle exec rake release`, which will create a git tag for the version,
 push git commits and tags, and push the `.gem` file to 
 [rubygems.org](https://rubygems.org).
 

data/lib/resque/kubernetes/context_factory.rb

@@ -32,17 +32,38 @@ module Resque
 
         def kubectl_context
           config = Kubeclient::Config.read(kubeconfig)
+          auth_options = config.context.auth_options
+
+          auth_options = google_default_application_credentials(config) if auth_options.empty?
+
           Kubeclient::Config::Context.new(
               config.context.api_endpoint,
               config.context.api_version,
               config.context.ssl_options,
-              use_default_gcp: true
+              auth_options
           )
         end
 
         def kubeconfig
           File.join(ENV["HOME"], ".kube", "config")
         end
+
+        # TODO: Move this logic to kubeclient. See abonas/kubeclient#213
+        def google_default_application_credentials(config)
+          return unless defined?(Google) && defined?(Google::Auth)
+
+          _cluster, user = config.send(:fetch_context, config.instance_variable_get(:@kcfg)["current-context"])
+          return {} unless user["auth-provider"] && user["auth-provider"]["name"] == "gcp"
+
+          {bearer_token: new_google_token}
+        end
+
+        def new_google_token
+          scopes = ["https://www.googleapis.com/auth/cloud-platform"]
+          authorization = Google::Auth.get_application_default(scopes)
+          authorization.apply({})
+          authorization.access_token
+        end
       end
     end
   end

data/lib/resque/kubernetes/job.rb

@@ -6,39 +6,83 @@ module Resque
   module Kubernetes
     # Resque hook to autoscale Kubernetes Jobs for workers.
     #
-    # To use, extend your Resque job class with this module and then define a
-    # class method `job_manifest` that produces the Kubernetes Job manifest.
+    # To use with pure Resque, extend your Resque job class with this module
+    # and then define a class method `job_manifest` that produces the
+    # Kubernetes Job manifest.
     #
-    # Example:
+    # To use with ActiveJob, include this module in your ActiveJob class
+    # and then define an instance method `job_manifest` that produces the
+    # Kubernetes Job manifest.
+    #
+    # Example (pure Resque):
     #
     #     class ResourceIntensiveJob
     #       extend Resque::Kubernetes::Job
+    #       class << self
+    #         def perform
+    #           # ... your existing code
+    #         end
+    #
+    #         def job_manifest
+    #           YAML.safe_load(
+    #             <<~MANIFEST
+    #             apiVersion: batch/v1
+    #               kind: Job
+    #               metadata:
+    #                 name: worker-job
+    #               spec:
+    #                 template:
+    #                   metadata:
+    #                     name: worker-job
+    #                   spec:
+    #                     containers:
+    #                     - name: worker
+    #                       image: us.gcr.io/project-id/some-resque-worker
+    #                       env:
+    #                       - name: QUEUE
+    #                         value: high-memory
+    #             MANIFEST
+    #           )
+    #         end
+    #       end
+    #     end
     #
+    # Example (ActiveJob backed by Resque):
+    #
+    #     class ResourceIntensiveJob < ApplicationJob
+    #       include Resque::Kubernetes::Job
     #       def perform
     #         # ... your existing code
     #       end
     #
     #       def job_manifest
-    #         <<-EOD
-    #     apiVersion: batch/v1
-    #     kind: Job
-    #     metadata:
-    #       name: worker-job
-    #     spec:
-    #       template:
-    #         metadata:
-    #           name: worker-job
-    #         spec:
-    #           containers:
-    #           - name: worker
-    #             image: us.gcr.io/project-id/some-resque-worker
-    #             env:
-    #             - name: QUEUE
-    #               value: high-memory
-    #         EOD
+    #         YAML.safe_load(
+    #           <<~MANIFEST
+    #           apiVersion: batch/v1
+    #             kind: Job
+    #              metadata:
+    #                name: worker-job
+    #              spec:
+    #                template:
+    #                  metadata:
+    #                    name: worker-job
+    #                  spec:
+    #                    containers:
+    #                    - name: worker
+    #                      image: us.gcr.io/project-id/some-resque-worker
+    #                      env:
+    #                      - name: QUEUE
+    #                        value: high-memory
+    #           MANIFEST
+    #         )
     #       end
     #     end
     module Job
+      def self.included(base)
+        return unless base.respond_to?(:before_enqueue)
+        base.before_enqueue :before_enqueue_kubernetes_job
+      end
+
       # A before_enqueue hook that adds worker jobs to the cluster.
       def before_enqueue_kubernetes_job(*_)
         if defined? Rails

data/lib/resque/kubernetes/version.rb

@@ -2,6 +2,6 @@
 
 module Resque
   module Kubernetes
-    VERSION = "0.5.0"
+    VERSION = "0.6.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: resque-kubernetes
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Jeremy Wadsack
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-01 00:00:00.000000000 Z
+date: 2018-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -127,7 +127,6 @@ files:
 - ".rubocop.yml"
 - ".ruby-gemset"
 - ".ruby-version"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.3.1
-before_install: gem install bundler -v 1.13.6