active_job_k8s 0.2.0 → 1.0.0.pre.beta.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57feaec27f10c9503c7e9455b30526d080b446eb9194f93a7a20e8830b3f3ceb
-  data.tar.gz: a1447113bfcdae14dae4e870e859df4997ae1ab8d6a01dc3dd11351301a7a165
+  metadata.gz: d5b40e92ba26450d0cb797737e03552b8f89fea8dc1f4ae9da53d924b34e23ad
+  data.tar.gz: 6d30c4b30a4a1bf180e5617c8ce4582f27862d80a2ed2b1959dc1c57f782d123
 SHA512:
-  metadata.gz: feb62f548aaeffade90f53014060463959c3d672d674f348a82474271291b2f7d00e3bb68a6398a26ff7a77de04b7d137df633ad42141bd1623fbc2b785024bf
-  data.tar.gz: 203f78073d6ce7667ab48facf91262b6cab587142ce8afe1b557220e64dcd574ff093ee160e38e6f1eccbe1b4d23177718b524b0bd13eb548ce75282f35f133b
+  metadata.gz: a1f159b9d818fc7f1e94f19cd69ffa6288a19e453c444a1fc6ff5470613f46366b0a3a4cef266c488ae7062f0a93ff8872cced39182e0b0feda9f57c3de90318
+  data.tar.gz: 3042cee4f989340e528d5f2d153fc954489e43edd0c152fc32255ff717bdeeceb254db5db45c48740df401c0dda1eb5e58a3910ca44f507251bd1bfa3c7e9c51

data/.github/workflows/ruby.yml

@@ -21,11 +21,14 @@ jobs:
 
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        ruby-version: ['2.6', '2.7']
+        ruby-version: [ '2.7','3.0','3.1']
 
     steps:
     - uses: actions/checkout@v3
+    - name: Remove Gemfile.lock
+      run: rm -fr Gemfile.lock
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# V 0.2.0
+- change position of default manifest
+- implementation of delayed job to be scheduled in a specific moment 
+
 # V 0.1.0
 - first release as POC
 - base tests

data/Gemfile

@@ -4,7 +4,7 @@ source "https://rubygems.org"
 gemspec
 
 gem "rails","~> 6.1"
-gem "rake", "~> 12.0"
+gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
 
 gem 'simplecov', require: false

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    active_job_k8s (0.1.0)
+    active_job_k8s (0.2.0)
+      http (>= 5.0)
       kubeclient (~> 4.0)
       rails (>= 6.0, < 8)
 
@@ -72,57 +73,71 @@ GEM
     builder (3.2.4)
     concurrent-ruby (1.1.10)
     crass (1.0.6)
+    date (3.3.3)
     diff-lcs (1.5.0)
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    erubi (1.11.0)
+    erubi (1.12.0)
     ffi (1.15.5)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
     globalid (1.0.0)
       activesupport (>= 5.0)
-    http (4.4.1)
-      addressable (~> 2.3)
+    http (5.1.1)
+      addressable (~> 2.8)
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
-      http-parser (~> 1.2.0)
+      llhttp-ffi (~> 0.4.0)
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
-    http-parser (1.2.3)
-      ffi-compiler (>= 1.0, < 2.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     jsonpath (1.1.2)
       multi_json
-    kubeclient (4.10.1)
-      http (>= 3.0, < 5.0)
+    kubeclient (4.11.0)
+      http (>= 3.0, < 6.0)
       jsonpath (~> 1.0)
       recursive-open-struct (~> 1.1, >= 1.1.1)
       rest-client (~> 2.0)
-    loofah (2.19.0)
+    llhttp-ffi (0.4.0)
+      ffi-compiler (~> 1.0)
+      rake (~> 13.0)
+    loofah (2.19.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    mail (2.7.1)
+    mail (2.8.0)
       mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
     marcel (1.0.2)
     method_source (1.0.0)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
     mini_mime (1.1.2)
-    minitest (5.16.3)
+    minitest (5.17.0)
     multi_json (1.15.0)
+    net-imap (0.3.4)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.1)
+      timeout
+    net-smtp (0.3.3)
+      net-protocol
     netrc (0.11.0)
     nio4r (2.5.8)
-    nokogiri (1.13.9-x86_64-linux)
+    nokogiri (1.13.10-x86_64-linux)
       racc (~> 1.4)
-    public_suffix (5.0.0)
-    racc (1.6.0)
-    rack (2.2.4)
+    public_suffix (5.0.1)
+    racc (1.6.2)
+    rack (2.2.5)
     rack-test (2.0.2)
       rack (>= 1.3)
     rails (6.1.7)
@@ -143,15 +158,15 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.3)
-      loofah (~> 2.3)
+    rails-html-sanitizer (1.4.4)
+      loofah (~> 2.19, >= 2.19.1)
     railties (6.1.7)
       actionpack (= 6.1.7)
       activesupport (= 6.1.7)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
-    rake (12.3.3)
+    rake (13.0.6)
     recursive-open-struct (1.1.3)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
@@ -159,20 +174,20 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rexml (3.2.5)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.1)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.1)
-    simplecov (0.21.2)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
@@ -181,14 +196,15 @@ GEM
       simplecov (~> 0.19)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    sprockets (4.1.1)
+    sprockets (4.2.0)
       concurrent-ruby (~> 1.0)
-      rack (> 1, < 3)
+      rack (>= 2.2.4, < 4)
     sprockets-rails (3.4.2)
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
     thor (1.2.1)
+    timeout (0.3.1)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
     unf (0.1.4)
@@ -197,7 +213,7 @@ GEM
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.6.1)
+    zeitwerk (2.6.6)
 
 PLATFORMS
   x86_64-linux
@@ -205,7 +221,7 @@ PLATFORMS
 DEPENDENCIES
   active_job_k8s!
   rails (~> 6.1)
-  rake (~> 12.0)
+  rake (~> 13.0)
   rspec (~> 3.0)
   simplecov
   simplecov-cobertura (~> 2.1.0)

data/README.md

@@ -1,4 +1,4 @@
-# ActiveJobK8s
+# ActiveJobK8s [![Gem Version](https://badge.fury.io/rb/active_job_k8s.svg)](https://badge.fury.io/rb/active_job_k8s)
 
 WIP gem to make active job work with kubernetes jobs.
 
@@ -6,7 +6,7 @@ Roadmap for V1.0:
 - [x] ActiveJob.perform_later create a Job in k8s that will execute the job
 - [x] ActiveJob.perform_later with delay create a Job in k8s in suspended mode, 
       a task will enable it as soon as the time is reached
-- [ ] Limiting the number of concurrent jobs (if there are more jobs they will be created in suspended mode)
+- [x] Limiting the number of concurrent jobs (if there are more jobs they will be created in suspended mode)
 
 Future:
 - [ ] Metrics (because everyone like metrics)
@@ -117,9 +117,8 @@ rules:
     verbs:
       - get
       - list
-      - watch
       - create
-      - delete
+      - patch
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

data/active_job_k8s.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.summary       = "ActiveJob adapter for kubernetes job"
   spec.homepage      = "https://github.com/oniram88/active_job_k8s"
   spec.license       = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.6.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'kubeclient', '~> 4.0' #https://github.com/ManageIQ/kubeclient
+  spec.add_dependency 'http', '>= 5.0' # dipendenza di kubeclient, ma non si compila correttamente con versione di ruby > di 2.7
   spec.add_dependency 'rails', '>= 6.0', '<8'
 
 end

data/bin/setup

@@ -5,7 +5,7 @@ set -vx
 
 bundle install
 
-# Prepariamo l'applicazione esempio
+# Create dummy application
 rm -fr spec/dummy
 bundle exec rails new spec/dummy --skip-git --skip-javascript --skip-hotwire --skip-jbuilder --skip-test --skip-bootsnap --force
 
@@ -15,6 +15,7 @@ sed -i 's/2.7.4/>= 2.7.4/' spec/dummy/Gemfile
 sed -i '/end/i root "main#index"' spec/dummy/config/routes.rb
 sed -i '/root "main#index"/i get  "create_job", to:  "main#create_job", as: "create_job"' spec/dummy/config/routes.rb
 sed -i '/root "main#index"/i get  "send_email", to:  "main#send_email", as: "send_email"' spec/dummy/config/routes.rb
+sed -i '/root "main#index"/i   get "health", to: "main#health"' spec/dummy/config/routes.rb
 
 echo '
 class MainController < ApplicationController
@@ -36,6 +37,10 @@ class MainController < ApplicationController
      redirect_to root_path
    end
 
+   def health
+     head :ok
+   end
+
    def send_email
      SubscriptionMailer.notify.deliver_later
      redirect_to root_path
@@ -117,6 +122,7 @@ class HelloWorldJob < ApplicationJob
 
   def perform(*args)
     Rails.logger.debug "Hello World"
+    sleep (SecureRandom.random_number * 50).to_i # simulate operations
     FileUtils.touch(Rails.root.join("tmp","#{SecureRandom.hex}.test"))
   end
 end
@@ -146,7 +152,7 @@ RUN bundle install
 
 COPY . .
 
-CMD ["/bin/sh", "-ec", "bundle install && bundle exec rails s -b 0.0.0.0 -p 3000"]' > spec/dummy/Dockerfile
+CMD ["/bin/sh", "-ec", "rm -fr /usr/src/app/spec/dummy/tmp/pids/* && bundle install && bundle exec rails s -b 0.0.0.0 -p 3000"]' > spec/dummy/Dockerfile
 
 docker build -t activejobk8s:0.1.0 spec/dummy/.
 
@@ -191,6 +197,12 @@ spec:
     volumeMounts:
     - mountPath: /usr/src/app
       name: application-volume
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: 3000
+      initialDelaySeconds: 120
+      periodSeconds: 3
   volumes:
   - name: application-volume
     hostPath:
@@ -210,9 +222,8 @@ rules:
     verbs:
       - get
       - list
-      - watch
       - create
-      - delete
+      - patch
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

data/bin/teardown

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+rm -fr spec/dummy
+kind delete cluster -n active-job-cluster

data/lib/active_job_k8s/scheduler.rb

@@ -11,17 +11,75 @@ module ActiveJobK8s
     #@return [Hash]
     attr_reader :default_manifest
 
-    # @param [Hash{ kubeclient_context: Kubeclient::Config::Context, default_manifest: Hash }] opts
-    def initialize(**opts)
-      raise "No KubeClientContext given" if opts[:kubeclient_context].nil?
+    # @param [Kubeclient::Config::Context] kubeclient_context
+    # @param [Hash] default_manifest
+    # @param [Integer] max_concurrent_jobs default 5
+    def initialize(kubeclient_context:, default_manifest: {}, max_concurrent_jobs: 5)
       # or to use a specific context, by name:
-      @kubeclient_context = opts[:kubeclient_context]
-      @default_manifest = opts[:default_manifest] || {}
-
+      @kubeclient_context = kubeclient_context
+      @default_manifest = default_manifest
+      @max_concurrent_jobs = max_concurrent_jobs
     end
 
     def create_job(job, scheduled_at: nil)
 
+      ##
+      # Make the job scheduled_at if there are not enought slot available
+      if scheduled_at.nil? and active_jobs.length>=@max_concurrent_jobs
+        scheduled_at = Time.now
+      end
+      _create_job(job, scheduled_at: scheduled_at)
+    end
+
+    def self.execute_job
+      ActiveJob::Base.execute(JSON.parse(ENV['SERIALIZED_JOB']))
+    end
+
+    ##
+    # Un-suspend jobs if the `scheduled at` is outdated, limited to max allowed concurrent
+    def un_suspend_jobs
+
+      to_activate_jobs = @max_concurrent_jobs - active_jobs.size
+      to_activate_jobs = 0 if to_activate_jobs < 0
+      # ::Rails.logger.debug { "Slot liberi: #{to_activate_jobs} - Job attivi: #{active_jobs.size} - Job in attesa: #{suspended_jobs.length}" }
+      suspended_jobs.select { |sj|
+        scheduled_at = Time.at(sj.metadata.annotations.scheduled_at.to_f)
+        Time.now > scheduled_at and sj.spec.suspend
+      }.take(to_activate_jobs).each do |sj|
+        client.patch_job(sj.metadata.name, { spec: { suspend: false } }, sj.metadata.namespace).inspect
+      end
+    end
+
+    protected
+
+    def client
+      @client ||= Kubeclient::Client.new(@kubeclient_context.api_endpoint + '/apis/batch',
+                                         @kubeclient_context.api_version || 'v1',
+                                         ssl_options: @kubeclient_context.ssl_options,
+                                         auth_options: @kubeclient_context.auth_options)
+    end
+
+    private
+
+    ##
+    # Internal list of all suspended jobs
+    def suspended_jobs
+      client.get_jobs(namespace: kubeclient_context.namespace,
+                      label_selector: "activeJobK8s=scheduled",
+                      field_selector: 'status.successful!=1')
+    end
+
+    # @return [Array<Kubeclient::Resource>]
+    def active_jobs
+      client.get_jobs(namespace: kubeclient_context.namespace,
+                      label_selector: "activeJobK8s",
+                      field_selector: 'status.successful!=1').select do |j|
+        j.status.active.to_i == 1 and j.spec.suspend != "true"
+      end
+    end
+
+    def _create_job(job, scheduled_at: nil)
+
       serialized_job = JSON.dump(job.serialize)
 
       manifest = (job.respond_to?(:manifest) and job.manifest.is_a?(Hash) and !job.manifest.empty?) ? job.manifest : default_manifest
@@ -44,7 +102,7 @@ module ActiveJobK8s
           container.args = ["active_job_k8s:run_job"]
         end
       end
-      kube_job.spec.ttlSecondsAfterFinished = 300 #number of seconds the job will be erased
+      kube_job.spec.ttlSecondsAfterFinished = 300 # number of seconds the job will be erased
 
       kube_job.metadata.labels ||= {}
       if scheduled_at
@@ -57,39 +115,6 @@ module ActiveJobK8s
       client.create_job(kube_job)
     end
 
-    def self.execute_job
-      ActiveJob::Base.execute(JSON.parse(ENV['SERIALIZED_JOB']))
-    end
-
-    ##
-    # Un-suspend jobs if the scheduled at is outdated
-    def un_suspend_jobs
-      suspended_jobs.each do |sj|
-        scheduled_at = Time.at(sj.metadata.annotations.scheduled_at.to_f)
-        if Time.now > scheduled_at and sj.spec.suspend
-          client.patch_job(sj.metadata.name, { spec: { suspend: false } }, sj.metadata.namespace).inspect
-        end
-      end
-    end
-
-    protected
-
-    def client
-      @client ||= Kubeclient::Client.new(@kubeclient_context.api_endpoint + '/apis/batch',
-                                         @kubeclient_context.api_version || 'v1',
-                                         ssl_options: @kubeclient_context.ssl_options,
-                                         auth_options: @kubeclient_context.auth_options)
-    end
-
-    private
-
-    ##
-    # Internal list of all suspended jobs
-    def suspended_jobs
-      client.get_jobs(namespace: kubeclient_context.namespace,
-                      label_selector: "activeJobK8s=scheduled",
-                      field_selector: 'status.successful!=1')
-    end
   end
 
 end

data/lib/active_job_k8s/version.rb

@@ -1,3 +1,3 @@
 module ActiveJobK8s
-  VERSION = "0.2.0"
+  VERSION = "1.0.0-beta.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_job_k8s
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0.pre.beta.1
 platform: ruby
 authors:
 - Marino Bonetti
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-11-06 00:00:00.000000000 Z
+date: 2023-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kubeclient
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,7 @@ files:
 - bin/cluster.yaml
 - bin/console
 - bin/setup
+- bin/teardown
 - lib/active_job/queue_adapters/k8s_adapter.rb
 - lib/active_job_k8s.rb
 - lib/active_job_k8s/railtie.rb
@@ -88,14 +103,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.6.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.2.33
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: ActiveJob adapter for kubernetes job