unicorn_wrangler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b9bc650286ed195d169cc671badd483392764a89
+  data.tar.gz: 93edb5fc2480c7234f787e3bc69341eb952ad9c0
+SHA512:
+  metadata.gz: 947a5a76a418ad1ee71b18246b04d8a3264f1485426f5499424f346dbe596ffdd1b987e50486a376221fe01cfd00e31df27d5e6f3a18290936ad6a846d838c9c
+  data.tar.gz: 86bede236627be26a1ea6f85d1a3c577cac936dbabaeb97ab5e2dec3a9c62a015ec2c68333c998ea12620bf839e7de6470804e689d9f542620e5288152cf860a

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (C) 2013 Michael Grosser <michael@grosser.it>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/unicorn_wrangler/version.rb

@@ -0,0 +1,3 @@
+module UnicornWrangler
+  VERSION = "0.1.0"
+end

data/lib/unicorn_wrangler.rb

@@ -0,0 +1,143 @@
+# - kills worker when they use too much memory
+# - kills worker when they did too many requests (resets leaked memory)
+# - runs GC out of band (does not block requests)
+
+require 'benchmark'
+
+module UnicornWrangler
+  STATS_NAMESPACE = 'unicorn'
+
+  class << self
+    attr_reader :handlers
+
+    # called from unicorn config (usually config/unicorn.rb)
+    # high level interface to keep setup consistent / simple
+    # set values to false to disable
+    def setup(
+      kill_after_requests: 10000,
+      gc_after_request_time: 10,
+      kill_on_too_much_memory: {},
+      logger:,
+      stats: nil # provide a statsd client with your apps namespace to collect stats
+    )
+      logger.info "Sending stats to under #{stats.namespace}.#{STATS_NAMESPACE}" if stats
+      @handlers = []
+      @handlers << RequestKiller.new(logger, stats, kill_after_requests) if kill_after_requests
+      @handlers << OutOfMemoryKiller.new(logger, stats, kill_on_too_much_memory) if kill_on_too_much_memory
+      @handlers << OutOfBandGC.new(logger, stats, gc_after_request_time) if gc_after_request_time
+      Unicorn::HttpServer.prepend UnicornExtension
+    end
+
+    # called from the unicorn server after each request
+    def perform_request
+      returned = nil
+      @requests ||= 0
+      @requests += 1
+      @request_time ||= 0
+      @request_time += Benchmark.realtime { returned = yield }
+      returned
+    ensure
+      @handlers.each { |handler| handler.call(@requests, @request_time) }
+    end
+  end
+
+  module UnicornExtension
+    def process_client(*)
+      UnicornWrangler.perform_request { super }
+    end
+  end
+
+  class Killer
+    def initialize(logger, stats)
+      @logger = logger
+      @stats = stats
+    end
+
+    private
+
+    # Kills the server, thereby resetting @requests / @request_time in the UnicornWrangler
+    #
+    # Possible issue: kill_worker is not meant to kill the server pid ... might have strange side effects
+    def kill(reason, memory, requests, request_time)
+      if @stats
+        @stats.increment("unicorn.kill.#{reason}")
+
+        @stats.histogram('unicorn.kill.memory', request_time)
+        @stats.histogram('unicorn.kill.total_requests', requests)
+        @stats.histogram('unicorn.kill.total_request_time', request_time)
+      end
+
+      @logger.info "Killing unicorn worker ##{Process.pid} for #{reason}. Requests: #{requests}, Time: #{request_time}, Memory: #{memory}MB"
+
+      Process.kill(:QUIT, Process.pid)
+    end
+
+    # expensive, do not run on every request
+    def used_memory
+      `ps -o rss= -p #{Process.pid}`.to_i / 1024
+    end
+  end
+
+  class OutOfMemoryKiller < Killer
+    def initialize(logger, stats, max: 20, check_every: 250)
+      super(logger, stats)
+      @max = max
+      @check_every = check_every
+      @logger.info "Killing workers when using more than #{@max}MB"
+    end
+
+    def call(requests, request_time)
+      return unless (requests % @check_every).zero? # avoid overhead of checking memory too often
+      return unless (memory = used_memory) > @max
+      kill :memory, memory, requests, request_time
+    end
+  end
+
+  class RequestKiller < Killer
+    def initialize(logger, stats, max_requests)
+      super(logger, stats)
+      @max_requests = max_requests
+      @logger.info "Killing workers after #{@max_requests} requests"
+    end
+
+    def call(requests, request_time)
+      kill(:requests, used_memory, requests, request_time) if requests >= @max_requests
+    end
+  end
+
+  # Do not run GC inside of requests, but only after a certain time spent in requests
+  #
+  # Alternative:
+  # https://github.com/tmm1/gctools
+  # which is more sophisticated and will result in less time spent GCing and less overall memory needed
+  class OutOfBandGC
+    def initialize(logger, stats, max_request_time)
+      @logger = logger
+      @stats = stats
+      @max_request_time = max_request_time
+      GC.disable
+      @logger.info "Garbage collecting after #{@max_request_time}s of request processing time"
+      @gc_ran_at = 0
+    end
+
+    def call(_requests, request_time)
+      time_since_last_gc = request_time - @gc_ran_at
+      return unless time_since_last_gc >= @max_request_time
+      @gc_ran_at = request_time
+
+      time = Benchmark.realtime do
+        GC.enable
+        GC.start
+        GC.disable
+      end
+
+      time = (time * 1000).round # s -> ms
+      if @stats
+        @stats.increment("#{STATS_NAMESPACE}.oobgc.runs")
+        @stats.timing("#{STATS_NAMESPACE}.oobgc.time", time)
+      end
+      @logger.info "Garbage collecting: took #{time}ms"
+      true
+    end
+  end
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: unicorn_wrangler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michael Grosser
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-20 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: michael@grosser.it
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- lib/unicorn_wrangler.rb
+- lib/unicorn_wrangler/version.rb
+homepage: https://github.com/grosser/unicorn_wrangler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: 'Unicorn: out of band GC / restart on max memory bloat / restart after X
+  requests'
+test_files: []