sidekiq-max-jobs 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ae8b389a6961487e91003baca5d13e15237f3cff0732cd789a57ead24781deba
+  data.tar.gz: 3aaedb46b7abb7f78cb0794bb759561ea632b208787ae8218a9a75d8ac511856
+SHA512:
+  metadata.gz: 5e40e4105fededa4a8b4b3513673bd3fcbbdb83e6ae9c0cf064cfa30bc32d66ccca61f9dbcedbbce91ccd4c40312af6a5dc90bcc057f74e663d72c2cc47b848c
+  data.tar.gz: 632290222af419e9ddb70e86bb58adb20e1a4bd5123079f816d81fd03ad676e5c27939852b6c11378a09a733c451550cb78735ceb4ad49efebbe8197f06f3f01

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+
+.ruby-version
+
+Gemfile.lock

data/.rubocop.yml

@@ -0,0 +1,38 @@
+inherit_from: .rubocop_todo.yml
+
+require: 'rubocop-rspec'
+
+AllCops:
+  Exclude:
+    - 'bin/**/*'
+    - 'config/**/*'
+    - 'db/**/*'
+    - 'vendor/**/*'
+
+Layout/DotPosition:
+  Enabled: false
+
+Layout/SpaceInLambdaLiteral:
+  EnforcedStyle: require_space
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+
+Metrics/LineLength:
+  Max: 120
+
+Style/Documentation:
+  Enabled: false
+
+Style/DoubleNegation:
+  Enabled: false
+
+Style/NegatedIf:
+  Enabled: false
+
+Style/RegexpLiteral:
+  AllowInnerSlashes: true
+
+Style/SymbolArray:
+  EnforcedStyle: brackets

data/.rubocop_todo.yml

@@ -0,0 +1,36 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2020-06-09 22:11:35 -0500 using RuboCop version 0.85.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Lint/RescueException:
+  Exclude:
+    - 'lib/sidekiq/middleware/server/max_jobs.rb'
+
+# Offense count: 1
+# Configuration parameters: IgnoredMethods.
+Metrics/AbcSize:
+  Max: 19
+
+# Offense count: 1
+# Configuration parameters: CountComments, ExcludedMethods.
+# ExcludedMethods: refine
+Metrics/BlockLength:
+  Max: 26
+
+# Offense count: 1
+# Configuration parameters: CountComments, ExcludedMethods.
+Metrics/MethodLength:
+  Max: 17
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: use_perl_names, use_english_names
+Style/SpecialGlobalVars:
+  Exclude:
+    - 'sidekiq-max-jobs.gemspec'

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,78 @@
+sidekiq-max-jobs
+================
+
+A [Sidekiq](https://sidekiq.org/) server middleware. Requires `sidekiq >= 4.0.0`
+
+This gem provides the ability to configure the maximum number of jobs a `Worker`
+will process before terminating. For an environment running _Kubernetes_ this is
+a perfect addition because once the affected pod dies it will automatically be
+restarted resetting memory, database-connections, etc. with minimal interruption
+to processing throughput.
+
+Origin Story
+------------
+
+While working on a project for [HappyFunCorp](https://happyfuncorp.com/) we were
+dealing with unmanageable memory growth on our primary DB. We did all the
+regular things for _Sidekiq_ such as disabling prepared statements, running with
+uncached queries, etc. to no avail. After lots of Googling, like any dilligent
+developer does, we hit a dead-end. A number of people had seen this issue and
+reported it, yet there was no real guidance aside from "restart your workers
+periodically to free resources." We saw that this worked, however rather than
+setting up a CRON we decided to implement a middleware that gave each `Worker`
+the reigns at controlling its own fate. What started as a work-around, turned
+out to actually be a pretty good solution vs. switching to a different
+background-job processing framework. Given that others are facing the same / a
+similar issue, we wanted to give it back to the open-source community.
+
+Install & Quick Start
+---------------------
+
+To install:
+```bash
+$ gem install sidekiq-max-jobs
+```
+
+If you're using [Bundler](https://bundler.io/) to manage your dependencies you
+should add the following to your Gemfile:
+```ruby
+gem 'sidekiq-max-jobs'
+```
+
+Next, add the middleware to your `sidekiq` initializer (typically: config/initializers/sidekiq.rb)
+```ruby
+require 'sidekiq/middleware/server/max_jobs'
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Middleware::Server::MaxJobs
+  end
+end
+```
+
+If everything above is successful the next time you start your worker you will
+see a message like the following:
+```bash
+2020-06-10T00:23:31.789Z pid=73703 tid=oxifk6l13 INFO: Max-Jobs middleware enabled, shutting down pid: 73703 after: 100 job(s)
+```
+
+Configuration Options
+---------------------
+
+Above we covered how to get started, but that's only the beginning. There are a
+few configuration options available to you to customize the middleware's
+behavior (currently only configurable via the environment):
+
+* `MAX_JOBS`: The number of jobs to process before terminating (default: `100`)
+* `MAX_JOBS_JITTER`: Used as the upper-bound for calculating a random number
+between 1 and the value specified. This value is added to the `MAX_JOBS` value,
+mentioned above, to decrease the likelihood that all of your `Worker(s)`
+restart at / around the same time (default: `1`)
+
+Contributing
+------------
+
+1. Fork it (http://github.com/jzaleski/sidekiq-max-jobs/fork)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# Link the `[r]spec` task to the `test` task
+desc 'Run `rspec`'
+task test: :spec
+
+# Set the default task
+desc 'Run the default `Task`'
+task default: :test

data/VERSION

@@ -0,0 +1 @@
+0.0.2

data/lib/sidekiq/middleware/server/max_jobs.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Middleware
+    module Server
+      class MaxJobs
+        VERSION = File.read(
+          File.join(
+            File.dirname(__FILE__),
+            '..',
+            '..',
+            '..',
+            '..',
+            'VERSION'
+          )
+        ).strip
+
+        class << self
+          def counter
+            @counter ||= 0
+          end
+
+          def increment_counter!
+            @counter = counter.next
+          end
+
+          def log_info(message)
+            ::Sidekiq.logger.info(message) if defined?(::Sidekiq.logger)
+          end
+
+          def log_initialization!
+            log_info("Max-Jobs middleware enabled, shutting down pid: #{pid} after: #{max_jobs_with_jitter} job(s)")
+          end
+
+          def max_jobs
+            @max_jobs ||= (ENV['MAX_JOBS'] || 100).to_i
+          end
+
+          def max_jobs_jitter
+            @max_jobs_jitter ||= rand((ENV['MAX_JOBS_JITTER'] || 1).to_i)
+          end
+
+          def max_jobs_with_jitter
+            @max_jobs_with_jitter ||= max_jobs + max_jobs_jitter
+          end
+
+          def mutex
+            @mutex ||= ::Mutex.new
+          end
+
+          def pid
+            @pid ||= ::Process.pid
+          end
+        end
+
+        def call(
+          _,  # worker-instance
+          _,  # item
+          _   # queue
+        )
+          exception_raised = false
+          begin
+            yield
+          rescue Exception
+            # Set the `exception_raised` boolean to `true` so that the
+            # job-counter *is not* incremented in the `ensure` block
+            exception_raised = true
+            # Re-raise the `Exception` so that _Sidekiq_ can deal w/ it
+            raise
+          ensure
+            if !exception_raised
+              self.class.mutex.synchronize do
+                self.class.increment_counter!
+
+                if self.class.counter == self.class.max_jobs_with_jitter
+                  self.class.log_info("Max-Jobs quota met, shutting down pid: #{self.class.pid}")
+                  ::Process.kill('TERM', self.class.pid)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+::Sidekiq::Middleware::Server::MaxJobs.log_initialization!

data/sidekiq-max-jobs.gemspec

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+LIB_DIR = File.expand_path(__dir__, 'lib')
+$LOAD_PATH.unshift(LIB_DIR) unless $LOAD_PATH.include?(LIB_DIR)
+
+Gem::Specification.new do |s|
+  s.name = 'sidekiq-max-jobs'
+  s.version = File.read(File.join(File.dirname(__FILE__), 'VERSION')).strip
+  s.date = Time.now.strftime('%Y-%m-%d')
+  s.authors = ['Jonathan W. Zaleski']
+  s.email = ['JonathanZaleski@gmail.com']
+  s.summary = <<~SUMMARY
+    A simple plugin used to control the maximum number of jobs for a Sidekiq
+    worker to process before terminating
+  SUMMARY
+  s.description = <<~DESCRIPTION
+    This gem provides the ability to configure the maximum number of jobs a
+    Sidekiq worker will process before terminating. For an environment running
+    Kubernetes this is a perfect addition because once the affected pod dies it
+    will automatically be restarted [gracefully] resetting memory,
+    database-connections, etc. with minimal interruption to throughput
+  DESCRIPTION
+  s.homepage = 'http://github.com/jzaleski/sidekiq-max-jobs'
+  s.license = 'MIT'
+
+  s.files = `git ls-files`.split($/)
+  s.require_paths = %w[lib]
+
+  s.add_dependency('sidekiq', '>= 4.0.0', '< 7.0.0')
+
+  s.add_development_dependency('pry', '~> 0.13.0')
+  s.add_development_dependency('rake', '~> 13.0.0')
+  s.add_development_dependency('rspec', '~> 3.9.0')
+  s.add_development_dependency('rubocop', '~> 0.85.0')
+  s.add_development_dependency('rubocop-rspec', '~> 1.39.0')
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+# Configure `rspec`
+RSpec.configure { |config| config.fail_if_no_examples = true }

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-max-jobs
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Jonathan W. Zaleski
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-06-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.13.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.85.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.85.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.39.0
+description: |
+  This gem provides the ability to configure the maximum number of jobs a
+  Sidekiq worker will process before terminating. For an environment running
+  Kubernetes this is a perfect addition because once the affected pod dies it
+  will automatically be restarted [gracefully] resetting memory,
+  database-connections, etc. with minimal interruption to throughput
+email:
+- JonathanZaleski@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- lib/sidekiq/middleware/server/max_jobs.rb
+- sidekiq-max-jobs.gemspec
+- spec/.keep
+- spec/spec_helper.rb
+homepage: http://github.com/jzaleski/sidekiq-max-jobs
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: A simple plugin used to control the maximum number of jobs for a Sidekiq
+  worker to process before terminating
+test_files: []