unicorn-worker-killer-fork-for-newer-unicorn 0.4.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 62e7ae660fb9a5c3cc623ceee3c4bbb72fac9d848378d3ef0f95cc891e4038ca
+  data.tar.gz: cc239bfa14ed1fecde5cf0a489e7d190e95a36b5572557191ef8dcdcb79bc8e7
+SHA512:
+  metadata.gz: a64ea2d0eb718fcf54d0a433271bed669280e7ca317814f570a2450937a53d1a951f38ad3023f05612d506804df7ab0e91b00e1832b6a4c109c1afb5807b97c8
+  data.tar.gz: db9f5bfa157c5fd9d40538819f04f4a63d74c90cc16bb5463026624cd2e4ede353e0871b3cfce975deeb5e929a06d2db3ac2a0ba4e339f88ffa33bba62d01595

data/AUTHORS

@@ -0,0 +1,3 @@
+Kazuki Ohta <kazuki.ohta _at_ gmail.com>
+Sadayuki FURUHASHI <frsyuki _at_ gmail.com>
+Jonathan Clem <jonathan _at_ jclem.net>

data/ChangeLog

@@ -0,0 +1,56 @@
+Release 0.4.5 - 2021/04/13
+  * update for Unicorn 6
+
+Release 0.4.4 - 2015/11/13
+  * update for Unicorn 5
+
+Release 0.4.3 - 2013/12/23
+  * fix for occasional 'invalid argument' error
+
+Release 0.4.2 - 2013/09/23
+  * `verbose` options provides verbose logging
+
+Release 0.4.1 - 2013/03/12
+  * Kill attempts happen once per request, not in a loop
+
+Release 0.4.0 - 2013/03/12
+  * Send signals from inside a Thread (required for QUIT signal)
+  * Fix broken if/else in kill_self
+
+Release 0.3.6 - 2013/03/09
+  * do not require configuration
+  * fix missing randomize() in MaxRequests killer
+
+Release 0.3.5 - 2013/03/08
+  * configurable max QUIT, TERM, and sleep interval
+  * fix warning message of signal handling
+
+Release 0.3.4 - 2013/03/08
+  * try SIGQUIT first, rather than SIGTERM
+
+Release 0.3.3 - 2013/02/28
+  * fix for ruby 1.8
+
+Release 0.3.2 - 2013/02/03
+  * add an option to disable the check for both MaxRequests and MaxMemory
+
+Release 0.3.1 - 2013/01/13
+  * fix for ruby 1.9.2
+
+Release 0.3.0 - 2012/12/31
+  * fix for 'preload_app true'
+
+Release 0.2.1 - 2012/12/10
+  * fix critical namespace error, introduced at the previous version
+
+Release 0.2.0 - 2012/11/22
+  * change namespace to Unicorn::WorkerKiller from UnicornWorkerKiller
+
+Release 0.1.2 - 2012/11/17
+  * change unicorn version dependency to '~> 4'
+
+Release 0.1.1 - 2012/11/17
+  * fix homepage
+
+Release 0.1 - 2012/11/17
+  * first release

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,64 @@
+Unicorn is copyrighted free software by all contributors, see logs in
+revision control for names and email addresses of all of them.
+
+You can redistribute it and/or modify it under either the terms of the
+GNU General Public License (GPL) as published by the Free Software
+Foundation (FSF), version {3.0}[http://www.gnu.org/licenses/gpl-3.0.txt]
+or version {2.0}[http://www.gnu.org/licenses/gpl-2.0.txt]
+or the Ruby-specific license terms (see below).
+
+The unicorn project leader (Eric Wong) reserves the right to add future
+versions of the GPL (and no other licenses) as published by the FSF to
+the licensing terms.
+
+=== Ruby-specific terms (if you're not using the GPLv2 or GPLv3)
+
+  1. You may make and give away verbatim copies of the source form of the
+     software without restriction, provided that you duplicate all of the
+     original copyright notices and associated disclaimers.
+
+  2. You may modify your copy of the software in any way, provided that
+     you do at least ONE of the following:
+
+       a) place your modifications in the Public Domain or otherwise make them
+       Freely Available, such as by posting said modifications to Usenet or an
+       equivalent medium, or by allowing the author to include your
+       modifications in the software.
+
+       b) use the modified software only within your corporation or
+          organization.
+
+       c) rename any non-standard executables so the names do not conflict with
+       standard executables, which must also be provided.
+
+       d) make other distribution arrangements with the author.
+
+  3. You may distribute the software in object code or executable
+     form, provided that you do at least ONE of the following:
+
+       a) distribute the executables and library files of the software,
+       together with instructions (in the manual page or equivalent) on where
+       to get the original distribution.
+
+       b) accompany the distribution with the machine-readable source of the
+       software.
+
+       c) give non-standard executables non-standard names, with
+          instructions on where to get the original software distribution.
+
+       d) make other distribution arrangements with the author.
+
+  4. You may modify and include the part of the software into any other
+     software (possibly commercial).  But some files in the distribution
+     are not written by the author, so that they are not under this terms.
+
+  5. The scripts and library files supplied as input to or produced as
+     output from the software do not automatically fall under the
+     copyright of the software, but belong to whomever generated them,
+     and may be sold commercially, and may be aggregated with this
+     software.
+
+  6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+     IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+     WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+     PURPOSE.

data/README.md

@@ -0,0 +1,49 @@
+# unicorn-worker-killer
+
+[Unicorn](http://unicorn.bogomips.org/) is widely used HTTP-server for Rack applications. One thing we thought Unicorn missed, is killing the Unicorn workers based on the number of requests and consumed memories.
+
+`unicorn-worker-killer` gem provides automatic restart of Unicorn workers based on 1) max number of requests, and 2) process memory size (RSS), without affecting any requests. This will greatly improves site's stability by avoiding unexpected memory exhaustion at the application nodes.
+
+# Install
+
+No external process like `god` is required. Just install one gem: `unicorn-worker-killer`.
+
+    gem 'unicorn-worker-killer'
+
+# Usage
+
+Add these lines to your `config.ru`. (These lines should be added above the `require ::File.expand_path('../config/environment',  __FILE__)` line.
+
+    # Unicorn self-process killer
+    require 'unicorn/worker_killer'
+    
+    # Max requests per worker
+    use Unicorn::WorkerKiller::MaxRequests, 3072, 4096
+    
+    # Max memory size (RSS) per worker
+    use Unicorn::WorkerKiller::Oom, (192*(1024**2)), (256*(1024**2))
+
+This gem provides two modules.
+
+### `Unicorn::WorkerKiller::MaxRequests(max_requests_min=3072, max_requests_max=4096, verbose=false)`
+
+This module automatically restarts the Unicorn workers, based on the number of requests which worker processed.
+
+`max_requests_min` and `max_requests_max` specify the min and max of maximum requests per worker. The actual limit is decided by rand() between `max_requests_min` and `max_requests_max` per worker, to prevent all workers to be dead at the same time. Once the number exceeds the limit, that worker is automatically restarted.
+
+If `verbose` is set to true, then after every request, your log will show the requests left before restart.  This logging is done at the `info` level.
+
+### `Unicorn::WorkerKiller::Oom(memory_limit_min=(1024\*\*3), memory_limit_max=(2\*(1024\*\*3)), check_cycle = 16, verbose = false)`
+
+This module automatically restarts the Unicorn workers, based on its memory size.
+
+`memory_limit_min` and `memory_limit_max` specify the min and max of maximum memory in bytes per worker. The actual limit is decided by rand() between `memory_limit_min` and `memory_limit_max` per worker, to prevent all workers to be dead at the same time.  Once the memory size exceeds `memory_size`, that worker is automatically restarted.
+
+The memory size check is done in every `check_cycle` requests.
+
+If `verbose` is set to true, then every memory size check will be shown in your logs.   This logging is done at the `info` level.
+
+# Special Thanks
+
+- [@hotchpotch](http://github.com/hotchpotch/) for the [original idea](https://gist.github.com/hotchpotch/1258681)
+

data/Rakefile

@@ -0,0 +1,5 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.4.5

data/lib/unicorn/worker_killer/configuration.rb

@@ -0,0 +1,11 @@
+module Unicorn::WorkerKiller
+  class Configuration
+    attr_accessor :max_quit, :max_term, :sleep_interval
+
+    def initialize
+      self.max_quit = 10
+      self.max_term = 15
+      self.sleep_interval = 1
+    end
+  end
+end

data/lib/unicorn/worker_killer.rb

@@ -0,0 +1,114 @@
+require 'unicorn'
+require 'unicorn/worker_killer/configuration'
+require 'get_process_mem'
+
+module Unicorn::WorkerKiller
+  class << self
+    attr_accessor :configuration
+  end
+
+  # Kill the current process by telling it to send signals to itself. If
+  # the process isn't killed after `configuration.max_quit` QUIT signals,
+  # send TERM signals until `configuration.max_term`. Finally, send a KILL
+  # signal. A single signal is sent per request.
+  # @see http://unicorn.bogomips.org/SIGNALS.html
+  def self.kill_self(logger, start_time)
+    alive_sec = (Time.now - start_time).round
+    worker_pid = Process.pid
+
+    @@kill_attempts ||= 0
+    @@kill_attempts += 1
+
+    sig = :QUIT
+    sig = :TERM if @@kill_attempts > configuration.max_quit
+    sig = :KILL if @@kill_attempts > configuration.max_term
+
+    logger.warn "#{self} send SIG#{sig} (pid: #{worker_pid}) alive: #{alive_sec} sec (trial #{@@kill_attempts})"
+    Process.kill sig, worker_pid
+  end
+
+  module Oom
+    # Killing the process must be occurred at the outside of the request. We're
+    # using similar techniques used by OobGC, to ensure actual killing doesn't
+    # affect the request.
+    #
+    # @see https://github.com/defunkt/unicorn/blob/master/lib/unicorn/oob_gc.rb#L40
+    def self.new(app, memory_limit_min = (1024**3), memory_limit_max = (2*(1024**3)), check_cycle = 16, verbose = false)
+      ObjectSpace.each_object(Unicorn::HttpServer) do |s|
+        s.extend(self)
+        s.instance_variable_set(:@_worker_memory_limit_min, memory_limit_min)
+        s.instance_variable_set(:@_worker_memory_limit_max, memory_limit_max)
+        s.instance_variable_set(:@_worker_check_cycle, check_cycle)
+        s.instance_variable_set(:@_worker_check_count, 0)
+        s.instance_variable_set(:@_verbose, verbose)
+      end
+      app # pretend to be Rack middleware since it was in the past
+    end
+
+    def randomize(integer)
+      RUBY_VERSION > "1.9" ? Random.rand(integer.abs) : rand(integer)
+    end
+
+    def process_client(*client)
+      super(*client) # Unicorn::HttpServer#process_client
+      return if @_worker_memory_limit_min == 0 && @_worker_memory_limit_max == 0
+
+      @_worker_process_start ||= Time.now
+      @_worker_memory_limit ||= @_worker_memory_limit_min + randomize(@_worker_memory_limit_max - @_worker_memory_limit_min + 1)
+      @_worker_check_count += 1
+      if @_worker_check_count % @_worker_check_cycle == 0
+        rss = GetProcessMem.new.bytes
+        logger.info "#{self}: worker (pid: #{Process.pid}) using #{rss} bytes." if @_verbose
+        if rss > @_worker_memory_limit
+          logger.warn "#{self}: worker (pid: #{Process.pid}) exceeds memory limit (#{rss} bytes > #{@_worker_memory_limit} bytes)"
+          Unicorn::WorkerKiller.kill_self(logger, @_worker_process_start)
+        end
+        @_worker_check_count = 0
+      end
+    end
+  end
+
+  module MaxRequests
+    # Killing the process must be occurred at the outside of the request. We're
+    # using similar techniques used by OobGC, to ensure actual killing doesn't
+    # affect the request.
+    #
+    # @see https://github.com/defunkt/unicorn/blob/master/lib/unicorn/oob_gc.rb#L40
+    def self.new(app, max_requests_min = 3072, max_requests_max = 4096, verbose = false)
+      ObjectSpace.each_object(Unicorn::HttpServer) do |s|
+        s.extend(self)
+        s.instance_variable_set(:@_worker_max_requests_min, max_requests_min)
+        s.instance_variable_set(:@_worker_max_requests_max, max_requests_max)
+        s.instance_variable_set(:@_verbose, verbose)
+      end
+
+      app # pretend to be Rack middleware since it was in the past
+    end
+
+    def randomize(integer)
+      RUBY_VERSION > "1.9" ? Random.rand(integer.abs) : rand(integer)
+    end
+
+    def process_client(*client)
+      super(*client) # Unicorn::HttpServer#process_client
+      return if @_worker_max_requests_min == 0 && @_worker_max_requests_max == 0
+
+      @_worker_process_start ||= Time.now
+      @_worker_cur_requests ||= @_worker_max_requests_min + randomize(@_worker_max_requests_max - @_worker_max_requests_min + 1)
+      @_worker_max_requests ||= @_worker_cur_requests
+      logger.info "#{self}: worker (pid: #{Process.pid}) has #{@_worker_cur_requests} left before being killed" if @_verbose
+
+      if (@_worker_cur_requests -= 1) <= 0
+        logger.warn "#{self}: worker (pid: #{Process.pid}) exceeds max number of requests (limit: #{@_worker_max_requests})"
+        Unicorn::WorkerKiller.kill_self(logger, @_worker_process_start)
+      end
+    end
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration) if block_given?
+  end
+
+  self.configure
+end

data/unicorn-worker-killer.gemspec

@@ -0,0 +1,21 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "unicorn-worker-killer-fork-for-newer-unicorn"
+  gem.description = "Kill unicorn workers by memory and request counts"
+  gem.homepage    = "https://github.com/jrupinski/unicorn-worker-killer"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Kazuki Ohta", "Sadayuki Furuhashi", "Jonathan Clem"]
+  gem.email       = ["kazuki.ohta@gmail.com", "frsyuki@gmail.com", "jonathan@jclem.net"]
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.licenses    = ["GPLv2+", "Ruby 1.8"]
+  gem.require_paths = ['lib']
+  gem.add_dependency "unicorn",         [">= 4", "< 7"]
+  gem.add_dependency "get_process_mem", "~> 0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: unicorn-worker-killer-fork-for-newer-unicorn
+version: !ruby/object:Gem::Version
+  version: 0.4.5
+platform: ruby
+authors:
+- Kazuki Ohta
+- Sadayuki Furuhashi
+- Jonathan Clem
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: unicorn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: get_process_mem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Kill unicorn workers by memory and request counts
+email:
+- kazuki.ohta@gmail.com
+- frsyuki@gmail.com
+- jonathan@jclem.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- AUTHORS
+- ChangeLog
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- lib/unicorn/worker_killer.rb
+- lib/unicorn/worker_killer/configuration.rb
+- unicorn-worker-killer.gemspec
+homepage: https://github.com/jrupinski/unicorn-worker-killer
+licenses:
+- GPLv2+
+- Ruby 1.8
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.8
+specification_version: 4
+summary: Kill unicorn workers by memory and request counts
+test_files: []