balepc-zombie_passenger_killer 0.2.1.patched

data/Gemfile

@@ -0,0 +1,7 @@
+source :rubygems
+gemspec
+
+group :development do
+  gem 'rake'
+  gem 'rspec', '~>2'
+end

data/Gemfile.lock

@@ -0,0 +1,26 @@
+PATH
+  remote: .
+  specs:
+    zombie_passenger_killer (0.2.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    rake (0.9.2)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake
+  rspec (~> 2)
+  zombie_passenger_killer!

data/Rakefile

@@ -0,0 +1,21 @@
+require 'bundler/gem_tasks'
+
+task :default do
+  sh "rspec spec/"
+end
+
+rule /^version:bump:.*/ do |t|
+  sh "git status | grep 'nothing to commit'" # ensure we are not dirty
+  index = ['major', 'minor','patch'].index(t.name.split(':').last)
+  file = 'lib/zombie_passenger_killer/version.rb'
+
+  version_file = File.read(file)
+  old_version, *version_parts = version_file.match(/(\d+)\.(\d+)\.(\d+)/).to_a
+  version_parts[index] = version_parts[index].to_i + 1
+  version_parts[2] = 0 if index < 2 # remove patch for minor
+  version_parts[1] = 0 if index < 1 # remove minor for major
+  new_version = version_parts * '.'
+  File.open(file,'w'){|f| f.write(version_file.sub(old_version, new_version)) }
+
+  sh "bundle && git add #{file} Gemfile.lock && git commit -m 'bump version to #{new_version}'"
+end

data/Readme.md

@@ -0,0 +1,66 @@
+![Zombies on a train](http://dl.dropbox.com/u/2670385/Web/zombie.jpeg)
+
+Guaranteed zombie passengers death.
+
+ - passenger process no longer listed in passenger-status ? => Death
+ - high CPU load over long period (Optional) ? => Death
+
+strace of killed zombies is printed, so debugging is easier.
+
+(god/bluepill are not suited to monitor passenger apps because of ever-changing pids)
+
+Add passenger-status to `/etc/sudoers` or run with sudo.
+
+Install
+=======
+    sudo gem install zombie_passenger_killer
+
+Usage
+=====
+
+    zombie_passenger_killer [options]
+
+    Options:
+        -m, --max [SIZE]                 Max high CPU entries in history before killing (default: off)
+            --history [SIZE]             History size (default: 5)
+        -c, --cpu [PERCENT]              Mark as high CPU when above PERCENT (default: 70)
+        -g, --grace [SECONDS]            Wait SECONDS before hard-killing (-9) a process (default: 5)
+        -i, --interval [SECONDS]         Check every SECONDS (default: 10)
+        -p, --pattern [PATTERN]          Find processes with this pattern (default: ' Rack: ')
+        -r, --rvm-sudo                   Use `rvmsudo` to see passenger-status
+        -h, --help                       Show this
+        -v, --version                    Show Version
+        -t, --time                       Show time in output
+
+
+### Bluepill script
+
+    app.process("zombie_passenger_killer") do |process|
+      process.start_command = "zombie_passenger_killer --max 5 --history 10 --cpu 30 --interval 10"
+      process.stdout = process.stderr = "/var/log/autorotate/zombie_passenger_killer.log"
+      process.pid_file = "/var/run/zombie_passenger_killer.pid"
+      process.daemonize = true
+    end
+
+### Monit script
+
+    check process zombie_killer
+      with pidfile "/var/run/zombie_passenger_killer.pid"
+      start program = "/bin/bash -c 'export PATH=$PATH:/usr/local/bin HOME=/home;zombie_passenger_killer --max 5 --history 10 --cpu 30 --interval 10 &>/var/log/zombie_passenger_killer.log & &>/dev/null;echo $! > /var/run/zombie_passenger_killer.pid'"
+      stop program = "/bin/bash -c 'PIDF=/var/run/zombie_passenger_killer.pid;/bin/kill `cat $PIDF` && rm -f $PIDF'"
+      group zombie_killer
+
+### God script
+
+# TODO
+
+Author
+======
+
+###Contributors
+ - [Roman Heinrich](https://github.com/mindreframer)
+ - [Kevin Mullin](https://github.com/kmullin)
+
+[Michael Grosser](http://grosser.it)<br/>
+michael@grosser.it<br/>
+Hereby placed under public domain, do what you want, just do not hold me accountable...

data/bin/zombie_passenger_killer

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+require 'optparse'
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'zombie_passenger_killer'
+
+options = {}
+version = ZombiePassengerKiller::VERSION
+OptionParser.new do |opts|
+  opts.banner = <<BANNER
+Guaranteed zombie passengers death
+
+Usage:
+    zombie_passenger_killer [options]
+
+Options:
+BANNER
+  opts.on("-m", "--max [SIZE]", Integer, "Max high CPU entries in history before killing (default: off)") {|i| options[:max]=i }
+  opts.on("--history [SIZE]", Integer, "History size (default: 5)") {|i| options[:history]=i }
+  opts.on("-c", "--cpu [PERCENT]", Integer, "Mark as high CPU when above PERCENT (default: 70)") {|i| options[:cpu]=i }
+  opts.on("-g", "--grace [SECONDS]", Integer, "Wait SECONDS before hard-killing (-9) a process (default: 5)") {|i| options[:grace]=i }
+  opts.on("-i", "--interval [SECONDS]", Integer, "Check every SECONDS (default: 10)") {|i| options[:interval]=i }
+  opts.on("-p", "--pattern [PATTERN]", String, "Find processes with this pattern (default: ' Rack: ')") {|i| options[:pattern]=i }
+  opts.on("-n", "--no-daemon","Do not start daemon. Check once and exit") {|i| options[:no_daemon]=i }
+  opts.on("-r", "--rvm-sudo","Use `rvmsudo` to see passenger-status") {|i| options[:rvm_sudo]=i }
+  opts.on("-h", "--help","Show this") { puts opts; exit }
+  opts.on("-v", "--version","Show Version"){ puts version; exit }
+  opts.on("-t", "--timing","Show timing on output") { options[:show_times]=true }
+end.parse!
+
+$stdout.sync = true
+puts "PID: #{$$} Version: #{version} Started at #{Time.now}"
+
+$0 = File.basename(__FILE__) + '-' + version
+unless options[:no_daemon].nil?
+  ZombiePassengerKiller::Reaper.new(options).hunt_zombies
+else
+  ZombiePassengerKiller::Reaper.new(options).lurk
+end

data/lib/zombie_passenger_killer/reaper.rb

@@ -0,0 +1,91 @@
+module ZombiePassengerKiller
+  class Reaper
+
+    attr_accessor :out # overwriteable for tests
+
+    def initialize(options)
+      @history = {}
+      @history_entries = options[:history] || 5
+      @max_high_cpu = options[:max]
+      @high_cpu = options[:cpu] || 70
+      @grace_time = options[:grace] || 5
+      @pattern = options[:pattern] || ' Rack: '
+      @show_times = options[:show_times] || false
+      @interval = options[:interval] || 10
+      @strace_time = 5
+      @out = STDOUT
+      @rvm_sudo = options[:rvm_sudo] || false
+    end
+
+    def lurk
+      loop do
+        hunt_zombies
+        sleep @interval
+      end
+    rescue Interrupt
+      log "Exiting..."
+      raise $!
+    end
+
+    def store_current_cpu(processes)
+      keys_to_remove = @history.keys - processes.map{|x| x[:pid] }
+      keys_to_remove.each{|k| !@history.delete k }
+
+      processes.each do |process|
+        @history[process[:pid]] ||= []
+        @history[process[:pid]] << process[:cpu]
+        @history[process[:pid]] = @history[process[:pid]].last(@history_entries)
+      end
+    end
+
+    def get_strace(pid, time)
+      Process.getpgid(pid) rescue return 'No such process'
+      `( strace -p #{pid} 2>&1 ) & sleep #{time} ; kill $! 2>&1`
+    end
+
+    def hunt_zombies
+      active_pids_in_passenger_status = passenger_pids
+      active_processes_in_processlist = process_status
+      zombies = active_processes_in_processlist.map{|x| x[:pid] } - active_pids_in_passenger_status rescue Array.new
+
+      # kill processes with high CPU if user wants it
+      high_load = if @max_high_cpu
+        store_current_cpu active_processes_in_processlist
+        active_pids_in_passenger_status.select do |pid|
+          @history[pid].count{|x| x > @high_cpu } >= @max_high_cpu
+        end
+      else
+        []
+      end
+
+      (high_load + zombies).each do |pid|
+        kill_zombie pid
+      end
+    end
+
+    # return array of pids reported from passenger-status command, nil if passenger-status doesn't run
+    def passenger_pids
+      pids = %x(#{'rvmsudo ' if @rvm_sudo}passenger-status|grep PID).split("\n").map { |x| x.strip.match(/PID: \d*/).to_s.split[1].to_i }
+      pids if $?.exitstatus.zero?
+    end
+
+    def process_status
+      %x(ps -eo pid,pcpu,args|grep -v grep|grep '#{@pattern}').split("\n").map do |line|
+         values = line.strip.split[0..1]
+         {:pid => values.first.to_i, :cpu => values.last.to_f}
+      end
+    end
+
+    def kill_zombie(pid)
+      log "Killing passenger process #{pid}"
+      log get_strace(pid, @strace_time)
+      Process.kill('TERM', pid) rescue nil
+      sleep @grace_time # wait for it to die
+      Process.kill('KILL', pid) rescue nil
+    end
+
+    def log(msg)
+      @out.puts "#{@show_times ? "** [#{Time.now.strftime('%Y-%m-%d %H:%M:%S')}] #$$: " : ''}#{msg}"
+    end
+  end
+end

data/lib/zombie_passenger_killer/version.rb

@@ -0,0 +1,3 @@
+module ZombiePassengerKiller
+  Version = VERSION = '0.2.1.patched'
+end

data/lib/zombie_passenger_killer.rb

@@ -0,0 +1,4 @@
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'zombie_passenger_killer/version'
+require 'zombie_passenger_killer/reaper'

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift 'lib'
+require 'zombie_passenger_killer'

data/spec/zombie_passenger_killer_spec.rb

@@ -0,0 +1,178 @@
+require File.expand_path('spec/spec_helper')
+
+describe ZombiePassengerKiller do
+  let(:killer){
+    ZombiePassengerKiller::Reaper.new(@options || {}).tap do |k|
+      k.stub!(:passenger_pids).and_return([111])
+      k.out = StringIO.new
+    end
+  }
+
+  def output
+    killer.out.rewind
+    killer.out.read
+  end
+
+  it "has a VERSION" do
+    ZombiePassengerKiller::VERSION.should =~ /^\d+\.\d+\.\d+$/
+  end
+
+  describe "#hunt_zombies" do
+    it "does not kill anything by default" do
+      killer.should_not_receive(:kill_zombie)
+      killer.hunt_zombies
+    end
+
+    it "finds the right zombies" do
+      killer.stub!(:passenger_pids).and_return([123])
+      killer.stub!(:process_status).and_return([{:pid => 124, :cpu => 0}])
+      killer.should_receive(:kill_zombie).with(124)
+      killer.hunt_zombies
+    end
+
+    it "kills zombies with high cpu over max" do
+      @options = {:max => 1}
+      killer.stub!(:process_status).and_return([{:pid => 111, :cpu => 100}])
+      killer.should_receive(:kill_zombie).with(111)
+      killer.hunt_zombies
+    end
+
+    it "does not kills zombies with high cpu under max" do
+      @options = {:max => 2}
+      killer.stub!(:process_status).and_return([{:pid => 111, :cpu => 100}])
+      killer.should_not_receive(:kill_zombie).with(111)
+      killer.hunt_zombies
+    end
+
+    it "ignores high cpu levels in old history" do
+      @options = {:max => 2, :history => 2}
+      killer.should_not_receive(:kill_zombie).with(111)
+      killer.stub!(:process_status).and_return([{:pid => 111, :cpu => 100}])
+      killer.hunt_zombies
+      killer.stub!(:process_status).and_return([{:pid => 111, :cpu => 0}])
+      killer.hunt_zombies
+      killer.stub!(:process_status).and_return([{:pid => 111, :cpu => 100}])
+      killer.hunt_zombies
+    end
+
+    it "kills on high cpu levels in recent history" do
+      @options = {:max => 2, :history => 2}
+      killer.stub!(:process_status).and_return([{:pid => 111, :cpu => 100}])
+      killer.hunt_zombies
+      killer.should_receive(:kill_zombie).with(111)
+      killer.hunt_zombies
+    end
+  end
+
+  describe "#kill_zombies" do
+    before do
+      killer.instance_eval{
+        @grace_time = 0.1
+        @strace_time = 0.1
+      }
+    end
+
+    def pid_of(marker)
+      processes = `ps -ef | grep '#{marker}' | grep -v grep`
+      processes.strip.split("\n").last.split(/\s+/)[1].to_i
+    end
+
+    def start_bogus_process(options={})
+      marker = "TEST---#{rand(999999999999)}"
+      Thread.new do
+        `ruby -e 'at_exit{ puts "proper exit"; #{'sleep 10' if options[:hang]}}; sleep 10; puts "#{marker}"' 2>&1`
+      end
+      sleep 1 # give process time to spin up
+      pid_of(marker)
+    end
+
+    def process_alive?(pid)
+      Process.getpgid(pid)
+    rescue Errno::ESRCH
+      false
+    end
+
+    it "kills normal processes" do
+      pid = start_bogus_process
+      lambda{
+        killer.kill_zombie(pid)
+        sleep 0.1
+      }.should change{ process_alive?(pid) }
+    end
+
+    it "kills hanging processes" do
+      pid = start_bogus_process :hang => true
+      lambda{
+        killer.kill_zombie(pid)
+        sleep 0.1
+      }.should change{ process_alive?(pid) }
+    end
+
+    it "prints an strace of the process" do
+      pid = start_bogus_process
+      killer.kill_zombie(pid)
+      output.should include('attach:')
+    end
+
+    it "does not take a strace of a dead process" do
+      killer.kill_zombie(111)
+      output.should_not include('attach:')
+    end
+
+    it "does not fail with an unknown pid" do
+      killer.kill_zombie(111)
+      output.should include('No such process')
+    end
+  end
+
+  describe "#log" do
+    it "logs simple when :show_times is not given" do
+      killer.log "X"
+      output.should == "X\n"
+    end
+
+    it "logs simple when :show_times is not given" do
+      @options = {:show_times => true}
+      killer.log "X"
+      output.should include(Time.now.year.to_s)
+    end
+  end
+
+  describe "#lurk" do
+    it "sleeps after checking" do
+      killer.should_receive(:hunt_zombies)
+      killer.should_receive(:sleep).with(10).and_raise "LOOP-BREAKER"
+      lambda{
+        killer.lurk
+      }.should raise_error "LOOP-BREAKER"
+    end
+
+    it "calls sleep with the given interval" do
+      @options = {:interval => 5}
+      killer.stub(:hunt_zombies)
+      killer.should_receive(:sleep).with(5).and_raise "LOOP-BREAKER"
+      lambda{
+        killer.lurk
+      }.should raise_error "LOOP-BREAKER"
+    end
+
+    it "prints Exiting on Interrupt" do
+      killer.stub(:hunt_zombies)
+      killer.should_receive(:sleep).and_raise Interrupt.new
+      lambda{
+        killer.lurk
+      }.should raise_error Interrupt
+      output.should include("Exiting")
+    end
+  end
+
+  describe "cli" do
+    it "prints its version" do
+      `./bin/zombie_passenger_killer -v`.should =~ /^\d+\.\d+\.\d+$/m
+    end
+
+    it "prints help" do
+      `./bin/zombie_passenger_killer -h`.should include('Usage')
+    end
+  end
+end

data/zombie_passenger_killer.gemspec

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+require 'zombie_passenger_killer/version'
+
+Gem::Specification.new "balepc-zombie_passenger_killer", ZombiePassengerKiller::VERSION do |s|
+  s.summary = "Guaranteed zombie passengers death"
+  s.authors = ["Michael Grosser"]
+  s.email = "michael@grosser.it"
+  s.homepage = "http://github.com/grosser/zombie_passenger_killer"
+  s.files = `git ls-files`.split("\n")
+  s.executables = ["zombie_passenger_killer"]
+  s.license = "MIT"
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification 
+name: balepc-zombie_passenger_killer
+version: !ruby/object:Gem::Version 
+  hash: -1513943312
+  prerelease: 6
+  segments: 
+  - 0
+  - 2
+  - 1
+  - patched
+  version: 0.2.1.patched
+platform: ruby
+authors: 
+- Michael Grosser
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-03-14 00:00:00 Z
+dependencies: []
+
+description: 
+email: michael@grosser.it
+executables: 
+- zombie_passenger_killer
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Gemfile
+- Gemfile.lock
+- Rakefile
+- Readme.md
+- bin/zombie_passenger_killer
+- lib/zombie_passenger_killer.rb
+- lib/zombie_passenger_killer/reaper.rb
+- lib/zombie_passenger_killer/version.rb
+- spec/spec_helper.rb
+- spec/zombie_passenger_killer_spec.rb
+- zombie_passenger_killer.gemspec
+homepage: http://github.com/grosser/zombie_passenger_killer
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      hash: 25
+      segments: 
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: Guaranteed zombie passengers death
+test_files: []
+