unicorn_wrangler 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 59947ed43ab082da1be724e6a29d42b65127f90d
-  data.tar.gz: 27c530b49b897af2ee2b922a0df9e03fa929f190
+SHA256:
+  metadata.gz: cfcaf29cd13aac1fef51db90f04d6e531f8d7c8c3b944e02449c35dae44d8211
+  data.tar.gz: edc4413e67dad70cd6f18589499b06ac1bff9a6830a6494e3139a2ff6cb2cb18
 SHA512:
-  metadata.gz: 5a041d009ad555382389ec005e2b288d4874562dcd4c63b02410c4305f79872d124ee68fa30ea99bd957ff4fa58a7aadc2da43391be5c66a4d414d4827e20bea
-  data.tar.gz: 363370e818b00151b5a99412deff1990538d1fcc9f43b8d2428b15a58e66c2fbabe22a9a3d3327fea18c3f22dfc4136769b0d2268bfeea8ef3eb8472fb8f9abe
+  metadata.gz: 9a5b367df477b3ce85ab39d08cb07de5e21c207bc17b0a459e3e0e2f1124b8e83b03baac0ca9316680c1330ec989a05d2e727380bbe6a3bfb3ac9d2ae5fde633
+  data.tar.gz: 5f6675d7e9e086bb25bd1604949ad24326fc1e8a2f88c01e2f50cd570a3314e23edb73322cabd6cb696cf88e50e389a0ff9c82e44dbed2d8a04abdef0a012713

data/lib/unicorn_wrangler.rb

@@ -3,12 +3,14 @@
 # - runs GC out of band (does not block requests)
 
 require 'benchmark'
+require 'unicorn_wrangler/rss_reader'
 
 module UnicornWrangler
   STATS_NAMESPACE = 'unicorn'
 
   class << self
-    attr_reader :handlers
+    attr_reader :handlers, :requests, :request_time
+    attr_accessor :sending_myself_term
 
     # called from unicorn config (usually config/unicorn.rb)
     # high level interface to keep setup consistent / simple
@@ -17,6 +19,7 @@ module UnicornWrangler
       kill_after_requests: 10000,
       gc_after_request_time: 10,
       kill_on_too_much_memory: {},
+      map_term_to_quit: false,
       logger:,
       stats: nil # provide a statsd client with your apps namespace to collect stats
     )
@@ -25,9 +28,40 @@ module UnicornWrangler
       @handlers << RequestKiller.new(logger, stats, kill_after_requests) if kill_after_requests
       @handlers << OutOfMemoryKiller.new(logger, stats, kill_on_too_much_memory) if kill_on_too_much_memory
       @handlers << OutOfBandGC.new(logger, stats, gc_after_request_time) if gc_after_request_time
+
+      @hooks = {}
+      if map_term_to_quit
+        # - on heroku & kubernetes all processes get TERM, so we need to trap in master and worker
+        # - trapping has to be done in the before_fork since unicorn sets up it's own traps on start
+        # - we cannot write to logger inside of a trap, so need to spawn a new Thread
+        # - manual test: add a slow route + rails s + curl + pkill -TERM -f 'unicorn master' - request finished?
+        @hooks[:before_fork] = -> do
+          Signal.trap :TERM do
+            Thread.new { logger.info 'master intercepting TERM and sending myself QUIT instead' }
+            Process.kill :QUIT, Process.pid
+          end
+        end
+
+        @hooks[:after_fork] = ->(*) do
+          # Signal.trap returns the trap that unicorn set, which is an exit!(0) and calls that when sending myself term
+          previous_trap = Signal.trap :TERM do
+            if sending_myself_term
+              previous_trap.call
+            else
+              Thread.new { logger.info 'worker intercepting TERM and doing nothing. Wait for master to send QUIT' }
+            end
+          end
+        end
+      end
+
       Unicorn::HttpServer.prepend UnicornExtension
     end
 
+    def kill_worker
+      self.sending_myself_term = true # no need to clean up since we are dead after
+      Process.kill(:TERM, Process.pid)
+    end
+
     # called from the unicorn server after each request
     def perform_request
       returned = nil
@@ -39,9 +73,26 @@ module UnicornWrangler
     ensure
       @handlers.each { |handler| handler.call(@requests, @request_time) }
     end
+
+    def perform_hook(name)
+      if hook = @hooks[name]
+        hook.call
+      end
+    end
   end
 
   module UnicornExtension
+    # call our hook and the users hook since only a single hook can be configured at a time
+    # we need to call super so the @<hook> variables get set and unset properly in after_fork to not leak memory
+    [:after_fork, :before_fork].each do |hook|
+      define_method("#{hook}=") do |value|
+        super(->(*args) do
+          UnicornWrangler.perform_hook(hook)
+          value.call(*args)
+        end)
+      end
+    end
+
     def process_client(*)
       UnicornWrangler.perform_request { super }
     end
@@ -59,6 +110,7 @@ module UnicornWrangler
     def initialize(logger, stats)
       @logger = logger
       @stats = stats
+      @rss_reader = RssReader.new(logger: logger)
     end
 
     private
@@ -75,18 +127,18 @@ module UnicornWrangler
         @stats.histogram("#{STATS_NAMESPACE}.kill.total_request_time", request_time)
       end
 
-      report_status "Killing", reason, memory, requests, request_time
+      report_status "Killing", reason, memory, requests, request_time, :warn
 
-      Process.kill(:TERM, Process.pid)
+      UnicornWrangler.kill_worker
     end
 
-    # expensive, do not run on every request
+    # RSS memory in MB. Can be expensive, do not run on every request
     def used_memory
-      `ps -o rss= -p #{Process.pid}`.to_i / 1024
+      @rss_reader.rss
     end
 
-    def report_status(status, reason, memory, requests, request_time)
-      @logger.info "#{status} unicorn worker ##{Process.pid} for #{reason}. Requests: #{requests}, Time: #{request_time}, Memory: #{memory}MB"
+    def report_status(status, reason, memory, requests, request_time, log_level = :debug)
+      @logger.send log_level, "#{status} unicorn worker ##{Process.pid} for #{reason}. Requests: #{requests}, Time: #{request_time}, Memory: #{memory}MB"
     end
   end
 
@@ -152,7 +204,7 @@ module UnicornWrangler
         @stats.increment("#{STATS_NAMESPACE}.oobgc.runs")
         @stats.timing("#{STATS_NAMESPACE}.oobgc.time", time)
       end
-      @logger.info "Garbage collecting: took #{time}ms"
+      @logger.debug "Garbage collecting: took #{time}ms"
       true
     end
   end

data/lib/unicorn_wrangler/rss_reader.rb

@@ -0,0 +1,50 @@
+# Read RSS based on the OS we are in. When in linux, we can read the proc status file and parse out
+# the RSS which is much faster than forking+execing ps.
+module UnicornWrangler
+  class RssReader
+    LINUX = RbConfig::CONFIG['host_os'].start_with?('linux')
+    PS_CMD = 'ps -o rss= -p %d'.freeze
+    VM_RSS = /^VmRSS:\s+(\d+)\s+(\w+)/
+    UNITS  = {
+      b:  1024**0,
+      kb: 1024**1,
+      mb: 1024**2,
+      gb: 1024**3,
+      tb: 1024**4,
+    }.freeze
+
+    def initialize(logger:)
+      @logger = logger
+    end
+
+    # Returns RSS in megabytes; should work on Linux and Mac OS X
+    def rss(pid: Process.pid)
+      LINUX ? rss_linux(pid) : rss_posix(pid)
+    end
+
+    private
+
+    # Fork/exec ps and parse result.
+    # Should work on any system with POSIX ps.
+    # ~4ms
+    # returns kb but we want mb
+    def rss_posix(pid)
+      `#{PS_CMD % [pid]}`.to_i / 1024
+    end
+
+    # Read from /proc/$pid/status.  Linux only.
+    # ~100x faster and doesn't incur significant memory cost.
+    # file returns variable units, we want mb
+    def rss_linux(pid)
+      File.read("/proc/#{pid}/status").match(VM_RSS) do |match|
+        value, magnitude = match[1].to_i, UNITS.fetch(match[2].downcase.to_sym)
+
+        value * magnitude / UNITS.fetch(:mb)
+      end
+    rescue
+      # If the given pid is dead, file will not be found
+      @logger.warn 'Failed to read RSS from /proc, falling back to exec+ps' if @logger
+      rss_posix(pid)
+    end
+  end
+end

data/lib/unicorn_wrangler/version.rb

@@ -1,3 +1,3 @@
 module UnicornWrangler
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unicorn_wrangler
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-15 00:00:00.000000000 Z
+date: 2020-10-13 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: michael@grosser.it
@@ -18,6 +18,7 @@ extra_rdoc_files: []
 files:
 - MIT-LICENSE
 - lib/unicorn_wrangler.rb
+- lib/unicorn_wrangler/rss_reader.rb
 - lib/unicorn_wrangler/version.rb
 homepage: https://github.com/grosser/unicorn_wrangler
 licenses:
@@ -31,15 +32,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: 'Unicorn: out of band GC / restart on max memory bloat / restart after X