kuby-sidekiq 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16ba8d64983f10c83431ce728402e36c40664b0ba89ff55f5c3677fc2291bcfe
-  data.tar.gz: 158df48cc2d19408fe295fb745f561a8a93858cfec765fac3fdb6a06fe2cee8e
+  metadata.gz: 96b955437e20df29e41ac12d18bcb7ae804af4858a35e58c1a29b54b76b16ee7
+  data.tar.gz: 183546d8e03bba59e1baa60e253a6abf682dfbbb1d450e5c550109016e7f1c5f
 SHA512:
-  metadata.gz: 26e27f3fc64531c064e70596d384e2d6279c63141ea0f7f1c481dbccbd144a8749927878bd1b11e64a19644e2144d02c974e2fd133981ac282b2a5bc9672f7ac
-  data.tar.gz: 6b9f8ab50e6d994d7bbd2d8874ce1c84e7d21d41491e98017eac61d6f38b40cfef34331423b62b7bcc9027da82318aa09255c847c9cbcb286276ac64a463bfe7
+  metadata.gz: 64178547f2fa845ab5879cbe39d6d98d62e2fb56d8433fbc7748e69e93eff3502cd1c503ca563a270a233ee6b5d7cfe4145962817597eb1f52111daffe26a17f
+  data.tar.gz: 85516c3f5ba4c5f87ddf0e46dd13fe159eccf2d4ab86685840d7ff3e7ba3df3574df477ff498b6eea8ad0fd6a0b2bcbd1f2f22accdb568ac4f55de19a93c176a

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.5.0
+* Support for multiple configurable Sidekiq processes (#2, @zhall0624)
+
+## 0.4.0
+* Use kuby-redis v0.2, which uses the Spotahome Redis operator instead of KubeDB.
+
 ## 0.3.0
 * Conform to new plugin architecture.
 * Accept `environment` instead of `definition` instances.

data/Gemfile

@@ -6,6 +6,7 @@ gem 'kuby-core', path: '../kuby-core'
 gem 'sidekiq', '>= 6'
 
 group :development, :test do
+  gem 'sorbet-runtime'
   gem 'pry-byebug'
   gem 'rake'
 end

data/README.md

@@ -0,0 +1,106 @@
+**NOTE**: The documentation below refers to an unreleased version of kuby-sidekiq.
+
+## kuby-sidekiq
+
+Sidekiq plugin for [Kuby](https://github.com/getkuby/kuby-core).
+
+## Intro
+
+The Sidekiq plugin makes it easy to run a deployment of Sidekiq workers for your Rails app. Behind the scenes it uses [kuby-redis](https://github.com/getkuby/kuby-redis) to stand up an instance of Redis and Kubernetes deployments to start the desired number of workers.
+
+## Configuration
+
+Add the kuby-sidekiq and [sidekiq](https://github.com/mperham/sidekiq) gems to your Gemfile and run `bundle install`.
+
+Require the plugin in your kuby.rb file and configure it, eg:
+
+```ruby
+require 'kuby/sidekiq'
+
+Kuby.define(:production) do
+  kubernetes do
+
+    add_plugin(:sidekiq) do
+      replicas 2  # run two workers
+    end
+
+  end
+end
+```
+
+Next, run the setup command, eg:
+
+```bash
+bundle exec kuby -e production setup
+```
+
+## Connecting to Redis
+
+Add a Sidekiq initializer to your Rails app and tell Sidekiq how to connect to Redis:
+
+```ruby
+# config/initializers/sidekiq.rb
+
+if Rails.env.production?
+  require 'kuby'
+
+  Kuby.load!
+
+  Sidekiq.configure_server do |config|
+    config.redis = Kuby.environment.kubernetes.plugin(:sidekiq).connection_params
+  end
+
+  Sidekiq.configure_client do |config|
+    config.redis = Kuby.environment.kubernetes.plugin(:sidekiq).connection_params
+  end
+end
+```
+
+### Build and Deploy
+
+Now that Sidekiq has been installed and configured, build, push, and deploy your app the usual way, eg:
+
+```bash
+bundle exec kuby -e production build
+bundle exec kuby -e production push
+bundle exec kuby -e production deploy
+```
+
+## Advanced Configuration
+
+If you have a more complex setup for Sidekiq you can specify the processes you need to create. The options method allow you to
+provide command line arguments to Sidekiq in order to either specify a non-default `sidekiq.yml` file or queues for a process to
+run. The process name is required.
+
+```ruby
+require 'kuby/sidekiq'
+
+Kuby.define(:production) do
+  kubernetes do
+
+    add_plugin(:sidekiq) do
+      replicas 2  # sets the default number of replicas for each process
+
+
+      process('default') # sidekiq uses sidekiq.yml by default and has 2 replicas
+
+      process('slow_process') do
+        options ['-C', 'config/slow_sidekiq.yml']
+        replicas 4 # override default number of replicas
+      end
+
+      process('queue_processor') do
+        options ['-q', 'critical', '-q', 'less_critical']
+      end
+    end
+  end
+end
+```
+
+## License
+
+Licensed under the MIT license. See LICENSE for details.
+
+## Authors
+
+* Cameron C. Dutro: http://github.com/camertron

data/Rakefile

@@ -2,6 +2,7 @@ require 'bundler'
 require 'rspec/core/rake_task'
 require 'rubygems/package_task'
 
+require 'sorbet-runtime'
 require 'kuby/sidekiq'
 
 Bundler::GemHelper.install_tasks

data/kuby-sidekiq.gemspec

@@ -12,8 +12,9 @@ Gem::Specification.new do |s|
 
   s.platform = Gem::Platform::RUBY
 
-  s.add_dependency 'kuby-kube-db', '~> 0.4'
-  s.add_dependency 'kube-dsl', '~> 0.3'
+  s.add_dependency 'kuby-core', '>= 0.16.0', '< 1.0'
+  s.add_dependency 'kube-dsl', '~> 0.7'
+  s.add_dependency 'kuby-redis', '~> 0.2'
 
   s.require_path = 'lib'
   s.files = Dir['{lib,spec}/**/*', 'Gemfile', 'LICENSE', 'CHANGELOG.md', 'README.md', 'Rakefile', 'kuby-sidekiq.gemspec']

data/lib/kuby/sidekiq/plugin.rb

@@ -1,4 +1,6 @@
 require 'securerandom'
+require 'kuby/redis'
+require_relative 'sidekiq_process'
 
 module Kuby
   module Sidekiq
@@ -7,33 +9,55 @@ module Kuby
 
       ROLE = 'worker'.freeze
 
-      value_fields :replicas
+      value_field :replicas, default: 1
 
-      def url
-        @url ||= "redis://#{redis.metadata.name}:6379/0"
+      def processes
+        @processes ||= []
       end
 
-      def after_initialize
-        @replicas = 1
+      def connection_params
+        redis_instance.connection_params
+      end
+
+      def configure(&block)
+        instance_eval(&block) if block
       end
 
       def after_configuration
+        if processes.empty?
+          processes << SidekiqProcess.new(plugin: self, default_replicas: replicas)
+        end
+
+        environment.kubernetes.add_plugin(:redis) do
+          instance :sidekiq do
+            custom_config (custom_config || []).concat(['maxmemory-policy noeviction'])
+          end
+        end
+
         return unless rails_app
 
-        deployment.spec.template.spec.container(:worker).merge!(
-          rails_app.deployment.spec.template.spec.container(:web), fields: [:env_from]
-        )
+        processes.each do |process|
+          process.deployment.spec.template.spec.container(:worker).merge!(
+            rails_app.deployment.spec.template.spec.container(:web), fields: [:env_from]
+          )
+
+          if rails_app.manage_database? && database = Kuby::Plugins::RailsApp::Database.get(rails_app)
+            database.plugin.configure_pod_spec(process.deployment.spec.template.spec)
+          end
+        end
       end
 
       def before_deploy(manifest)
-        image_with_tag = "#{docker.metadata.image_url}:#{kubernetes.tag}"
-
-        deployment do
-          spec do
-            template do
-              spec do
-                container(:worker) do
-                  image image_with_tag
+        image_with_tag = "#{docker.image.image_url}:#{kubernetes.tag || Kuby::Docker::LATEST_TAG}"
+
+        processes.each do |process|
+          process.deployment do
+            spec do
+              template do
+                spec do
+                  container(:worker) do
+                    image image_with_tag
+                  end
                 end
               end
             end
@@ -42,11 +66,7 @@ module Kuby
       end
 
       def resources
-        @resources ||= [
-          service_account,
-          deployment,
-          redis
-        ]
+        @resources ||= [service_account, *processes.map(&:deployment)]
       end
 
       def service_account(&block)
@@ -68,103 +88,15 @@ module Kuby
         @service_account
       end
 
-      def deployment(&block)
-        context = self
-
-        @deployment ||= KubeDSL.deployment do
-          metadata do
-            name "#{context.selector_app}-sidekiq-#{ROLE}"
-            namespace context.namespace.metadata.name
-
-            labels do
-              add :app, context.selector_app
-              add :role, ROLE
-            end
-          end
-
-          spec do
-            replicas context.replicas
-
-            selector do
-              match_labels do
-                add :app, context.selector_app
-                add :role, ROLE
-              end
-            end
-
-            strategy do
-              type 'RollingUpdate'
-
-              rolling_update do
-                max_surge '25%'
-                max_unavailable 0
-              end
-            end
-
-            template do
-              metadata do
-                labels do
-                  add :app, context.selector_app
-                  add :role, ROLE
-                end
-              end
-
-              spec do
-                container(:worker) do
-                  name "#{context.selector_app}-sidekiq-#{ROLE}"
-                  image_pull_policy 'IfNotPresent'
-                  command %w(bundle exec sidekiq)
-                end
-
-                image_pull_secret do
-                  name context.kubernetes.registry_secret.metadata.name
-                end
-
-                restart_policy 'Always'
-                service_account_name context.service_account.metadata.name
-              end
-            end
-          end
-        end
-
-        @deployment.instance_eval(&block) if block
-        @deployment
-      end
-
-      def redis(&block)
-        context = self
-
-        @redis ||= Kuby::KubeDB.redis do
-          api_version 'kubedb.com/v1alpha1'
-
-          metadata do
-            name "#{context.selector_app}-sidekiq-redis"
-            namespace context.kubernetes.namespace.metadata.name
-          end
-
-          spec do
-            version '5.0.3-v1'
-            storage_type 'Durable'
-
-            storage do
-              storage_class_name context.storage_class_name
-              access_modes ['ReadWriteOnce']
-
-              resources do
-                requests do
-                  add :storage, '1Gi'
-                end
-              end
-            end
-          end
+      def process(name, &block)
+        SidekiqProcess.new(name: name, plugin: self, default_replicas: replicas).tap do |process|
+          process.instance_eval(&block) if block
+          processes << process
         end
-
-        @redis.instance_eval(&block) if block
-        @redis
       end
 
-      def storage_class_name
-        kubernetes.provider.storage_class_name
+      def redis_instance
+        kubernetes.plugin(:redis).instance(:sidekiq)
       end
 
       def kubernetes

data/lib/kuby/sidekiq/sidekiq_process.rb

@@ -0,0 +1,89 @@
+require 'securerandom'
+
+module Kuby
+  module Sidekiq
+    # This class creates a deployment for a Sidekiq Process. A process is a single instance
+    # of Sidekiq. Each instance can be provided command line options to specify concurrency,
+    # config files or queues.
+    # https://github.com/sidekiq/sidekiq/wiki/Best-Practices#4-use-precise-terminology
+    class SidekiqProcess
+      extend ::KubeDSL::ValueFields
+
+      ROLE='worker'
+
+      attr_reader :plugin, :name, :default_replicas
+
+      value_field :replicas, default: nil
+      value_field :options, default: []
+
+      def initialize(name: 'default', plugin:, default_replicas:)
+        @name = name
+        @plugin = plugin
+        @default_replicas = default_replicas
+      end
+
+      def deployment(&block)
+        context = self
+
+        @deployment ||= KubeDSL.deployment do
+          metadata do
+            name "#{context.plugin.selector_app}-sidekiq-#{ROLE}-#{context.name}"
+            namespace context.plugin.namespace.metadata.name
+
+            labels do
+              add :app, context.plugin.selector_app
+              add :role, ROLE
+            end
+          end
+
+          spec do
+            replicas (context.replicas || context.default_replicas)
+
+            selector do
+              match_labels do
+                add :app, context.plugin.selector_app
+                add :role, ROLE
+              end
+            end
+
+            strategy do
+              type 'RollingUpdate'
+
+              rolling_update do
+                max_surge '25%'
+                max_unavailable 0
+              end
+            end
+
+            template do
+              metadata do
+                labels do
+                  add :app, context.plugin.selector_app
+                  add :role, ROLE
+                end
+              end
+
+              spec do
+                container(:worker) do
+                  name "#{context.plugin.selector_app}-sidekiq-#{ROLE}-#{context.name}"
+                  image_pull_policy 'IfNotPresent'
+                  command ['bundle', 'exec', 'sidekiq', *context.options]
+                end
+
+                image_pull_secret do
+                  name context.plugin.kubernetes.registry_secret.metadata.name
+                end
+
+                restart_policy 'Always'
+                service_account_name context.plugin.service_account.metadata.name
+              end
+            end
+          end
+        end
+
+        @deployment.instance_eval(&block) if block
+        @deployment
+      end
+    end
+  end
+end

data/lib/kuby/sidekiq/version.rb

@@ -1,5 +1,5 @@
 module Kuby
   module Sidekiq
-    VERSION = '0.3.0'
+    VERSION = '0.5.0'
   end
 end

metadata

@@ -1,43 +1,63 @@
 --- !ruby/object:Gem::Specification
 name: kuby-sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Cameron Dutro
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-30 00:00:00.000000000 Z
+date: 2023-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: kuby-kube-db
+  name: kuby-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: kube-dsl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
-  name: kube-dsl
+  name: kuby-redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.2'
 description: Sidekiq plugin for Kuby.
 email:
 - camertron@gmail.com
@@ -48,10 +68,12 @@ files:
 - CHANGELOG.md
 - Gemfile
 - LICENSE
+- README.md
 - Rakefile
 - kuby-sidekiq.gemspec
 - lib/kuby/sidekiq.rb
 - lib/kuby/sidekiq/plugin.rb
+- lib/kuby/sidekiq/sidekiq_process.rb
 - lib/kuby/sidekiq/version.rb
 homepage: http://github.com/getkuby/kuby-sidekiq
 licenses: []
@@ -71,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.4.5
 signing_key:
 specification_version: 4
 summary: Sidekiq plugin for Kuby.