puma_worker_killer 0.0.7 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20389b0e2ba67ab795071554a0e52b6599cd3612
-  data.tar.gz: 0789e6a76e4c12394df935bd9b16b503d6990a62
+  metadata.gz: 83921642be90494f9eb9988caff6f2397d2ff761
+  data.tar.gz: 0427e91feb9357ab8637739fac169edab633c8a1
 SHA512:
-  metadata.gz: 62aa47f0f5032df0d0887e8001143059b4107fe1871932b545d4e56820c0012260ea73ffb77aed9be3fe83a1a3ec08b2f23633a05f96d8375885b3cdf1e05454
-  data.tar.gz: 30fedd4919738c70432d7de4861089c03212c651805d4f564cd83e16d46f3b1d9bfc60ce44ae03a71cd1dff15b7547fbfb757b6b26d5e894f577ab0765669cad
+  metadata.gz: f87c5688cb637837c82182cad0eb912a5bcdfde7aab9fc8bd892154d51cb5c78019ea85a9edcaceba2cbb103e8062e68d8c9e500b29073d32bf7b22fafad56cc
+  data.tar.gz: e05e97981e2adf0e319b0ba92ff19f77e84a4f6c47906b7114fda2cdc18dce44e3476dd033f1995e076a073f25986fa74e342ea658f54b8dd7ba4a822af97998

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.0
+
+- Emit extra data via `pre_term` callback before puma worker killer terminates a worker #49.
+
 ## 0.0.7
 
 - Logging is configurable #41

data/README.md

@@ -34,6 +34,8 @@ Then run `$ bundle install`
 A rolling restart will kill each of your workers on a rolling basis. You set the frequency which it conducts the restart. This is a simple way to keep memory down as Ruby web programs generally increase memory usage over time. If you're using Heroku [it is difficult to measure RAM from inside of a container accurately](https://github.com/schneems/get_process_mem/issues/7), so it is recommended to use this feature or use a [log-drain-based worker killer](https://github.com/arches/whacamole). You can enable roling restarts by running:
 
 ```ruby
+# config/puma.rb
+
 before_fork do
   require 'puma_worker_killer'
 
@@ -103,10 +105,30 @@ PumaWorkerKiller.config do |config|
   config.rolling_restart_frequency = 12 * 3600 # 12 hours in seconds
   config.reaper_status_logs = true # setting this to false will not log lines like:
   # PumaWorkerKiller: Consuming 54.34765625 mb with master and 2 workers.
+
+  config.pre_term = -> (worker) { puts "Worker #{worker.inspect} being killed" }
 end
 PumaWorkerKiller.start
 ```
 
+### pre_term
+
+`config.pre_term` will be called just prior to worker termination with the worker that is about to be terminated. This may be useful to use in keeping track of metrics, time of day workers are restarted, etc.
+
+By default Puma Worker Killer will emit a log when a worker is being killed
+
+```
+PumaWorkerKiller: Out of memory. 5 workers consuming total: 500 mb out of max: 450 mb. Sending TERM to pid 23 consuming 53 mb.
+```
+
+or
+
+```
+PumaWorkerKiller: Rolling Restart. 5 workers consuming total: 650mb mb. Sending TERM to pid 34.
+```
+
+However you may want to collect more data, such as sending an event to an error collection service like rollbar or airbrake. The `pre_term` lambda gets called before any worker is killed by PWK for any reason.
+
 ## Attention
 
 If you start puma as a daemon, to add puma worker killer config into puma config file, rather than into initializers:

data/lib/puma_worker_killer.rb

@@ -3,19 +3,20 @@ require 'get_process_mem'
 module PumaWorkerKiller
   extend self
 
-  attr_accessor :ram, :frequency, :percent_usage, :rolling_restart_frequency, :reaper_status_logs
+  attr_accessor :ram, :frequency, :percent_usage, :rolling_restart_frequency, :reaper_status_logs, :pre_term
   self.ram           = 512  # mb
   self.frequency     = 10   # seconds
   self.percent_usage = 0.99 # percent of RAM to use
   self.rolling_restart_frequency = 6 * 3600 # 6 hours in seconds
   self.reaper_status_logs = true
+  self.pre_term = lambda { |_| } # nop
 
   def config
     yield self
   end
 
-  def reaper(ram = self.ram, percent = self.percent_usage, reaper_status_logs = self.reaper_status_logs)
-    Reaper.new(ram * percent_usage, nil, reaper_status_logs)
+  def reaper(ram = self.ram, percent = self.percent_usage, reaper_status_logs = self.reaper_status_logs, pre_term = self.pre_term)
+    Reaper.new(ram * percent_usage, nil, reaper_status_logs, pre_term)
   end
 
   def start(frequency = self.frequency, reaper = self.reaper)

data/lib/puma_worker_killer/puma_memory.rb

@@ -12,6 +12,10 @@ module PumaWorkerKiller
       workers.size
     end
 
+    def term_worker(worker)
+      worker.term
+    end
+
     def term_largest_worker
       largest_worker.term
     #   Process.wait(largest_worker.pid)
@@ -78,4 +82,4 @@ module PumaWorkerKiller
       end
     end
   end
-end
+end

data/lib/puma_worker_killer/reaper.rb

@@ -1,9 +1,10 @@
 module PumaWorkerKiller
   class Reaper
-    def initialize(max_ram, master = nil, reaper_status_logs = true)
+    def initialize(max_ram, master = nil, reaper_status_logs = true, pre_term)
       @cluster = PumaWorkerKiller::PumaMemory.new(master)
       @max_ram = max_ram
       @reaper_status_logs = reaper_status_logs
+      @pre_term = pre_term
     end
 
     # used for tes
@@ -15,7 +16,18 @@ module PumaWorkerKiller
       return false if @cluster.workers_stopped?
       if (total = get_total_memory) > @max_ram
         @cluster.master.log "PumaWorkerKiller: Out of memory. #{@cluster.workers.count} workers consuming total: #{total} mb out of max: #{@max_ram} mb. Sending TERM to pid #{@cluster.largest_worker.pid} consuming #{@cluster.largest_worker_memory} mb."
-        @cluster.term_largest_worker
+
+        # Fetch the largest_worker so that both `@pre_term` and `term_worker` are called with the same worker
+        # Avoids a race condition where:
+        #   Worker A consume 100 mb memory
+        #   Worker B consume 99 mb memory
+        #   pre_term gets called with Worker A
+        #   A new request comes in, Worker B takes it, and consumes 101 mb memory
+        #   term_largest_worker (previously here) gets called and terms Worker B (thus not passing the about-to-be-terminated worker to `@pre_term`)
+        largest_worker = @cluster.largest_worker
+        @pre_term.call(largest_worker)
+        @cluster.term_worker(largest_worker)
+
       elsif @reaper_status_logs
         @cluster.master.log "PumaWorkerKiller: Consuming #{total} mb with master and #{@cluster.workers.count} workers."
       end

data/lib/puma_worker_killer/rolling_restart.rb

@@ -12,7 +12,7 @@ module PumaWorkerKiller
     def reap(wait_between_worker_kill = 60) # seconds
       return false unless @cluster.running?
       @cluster.workers.each do |worker, ram|
-        @cluster.master.log "PumaWorkerKiller: Rolling Restart. #{@cluster.workers.count} workers consuming total: #{ get_total_memory } mb out of max: #{@max_ram} mb. Sending TERM to pid #{worker.pid}."
+        @cluster.master.log "PumaWorkerKiller: Rolling Restart. #{@cluster.workers.count} workers consuming total: #{ get_total_memory } mb. Sending TERM to pid #{worker.pid}."
         worker.term
         sleep wait_between_worker_kill
       end

data/lib/puma_worker_killer/version.rb

@@ -1,3 +1,3 @@
 module PumaWorkerKiller
-  VERSION = "0.0.7"
+  VERSION = "0.1.0"
 end

data/test/fixtures/pre_term.ru

@@ -0,0 +1,8 @@
+load File.expand_path("../fixture_helper.rb", __FILE__)
+
+PumaWorkerKiller.config do |config|
+  config.pre_term = lambda { |worker| puts("About to terminate worker: #{worker.inspect}") }
+end
+PumaWorkerKiller.start
+
+run HelloWorldApp

data/test/puma_worker_killer_test.rb

@@ -33,6 +33,18 @@ class PumaWorkerKillerTest < Test::Unit::TestCase
     end
   end
 
+  def test_pre_term
+    file     = fixture_path.join("pre_term.ru")
+    port     = 0
+    command  = "bundle exec puma #{ file } -t 1:1 -w 2 --preload --debug -p #{ port }"
+    options  = { wait_for: "booted", timeout: 5, env: { "PUMA_FREQUENCY" => 1, 'PUMA_RAM' => 1} }
+
+    WaitForIt.new(command, options) do |spawn|
+      assert_contains(spawn, "Out of memory")
+      assert_contains(spawn, "About to terminate worker:") # defined in pre_term.ru
+    end
+  end
+
   def assert_contains(spawn, string)
     assert spawn.wait(string), "Expected logs to contain '#{string}' but it did not, contents: #{ spawn.log.read }"
   end
@@ -50,4 +62,3 @@ class PumaWorkerKillerTest < Test::Unit::TestCase
     end
   end
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puma_worker_killer
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.1.0
 platform: ruby
 authors:
 - Richard Schneeman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-13 00:00:00.000000000 Z
+date: 2017-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: puma
@@ -124,6 +124,7 @@ files:
 - test/fixtures/config/puma_worker_killer_start.rb
 - test/fixtures/default.ru
 - test/fixtures/fixture_helper.rb
+- test/fixtures/pre_term.ru
 - test/fixtures/rolling_restart.ru
 - test/puma_worker_killer_test.rb
 - test/test_helper.rb
@@ -147,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: If you have a memory leak in your web code puma_worker_killer can keep it
@@ -157,7 +158,7 @@ test_files:
 - test/fixtures/config/puma_worker_killer_start.rb
 - test/fixtures/default.ru
 - test/fixtures/fixture_helper.rb
+- test/fixtures/pre_term.ru
 - test/fixtures/rolling_restart.ru
 - test/puma_worker_killer_test.rb
 - test/test_helper.rb
-has_rdoc: