crash-watch 1.1.0

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Phusion
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,55 @@
+# Introduction
+
+* Do you have (server) processes that sometimes crash for mysterious reasons?
+* Can you not figure out why?
+* Do they not print any error messages to their log files upon crashing?
+* Are debuggers complicated, scary things that you'd rather avoid?
+
+`crash-watch` to the rescue! This little program will monitor a specified process and wait until it crashes. It will then print useful information such as its exit status, what signal caused it to abort, and its backtrace.
+
+## Installation
+
+    gem install crash-watch
+
+You must also have GDB installed. Mac OS X already has it by default. If you're on Linux, try one of these:
+
+    apt-get install gdb
+    yum install gdb
+
+## Sample usage
+
+    $ crash-watch <PID>
+    Monitoring PID <PID>...
+    (...some time later, <PID> exits...)
+    Process exited.
+    Exit code: 0
+    Backtrace:
+        Thread 1 (process 95205):
+        #0  0x00007fff87ea1db0 in _exit ()
+        No symbol table info available.
+        #1  0x000000010002a260 in ruby_stop ()
+        No symbol table info available.
+        #2  0x0000000100031a54 in ruby_run ()
+        No symbol table info available.
+        #3  0x00000001000009e4 in main ()
+        No symbol table info available.
+
+While monitoring the process, you may interrupt `crash-watch` by pressing Ctrl-C. `crash-watch` will then detach from the process, which will then continue normally. You may re-attach `crash-watch` later.
+
+## Goodie: GDB controller
+
+I've written a small library for controlling gdb, which `crash-watch` uses internally. With CrashWatch::GdbController you can send arbitrary commands to gdb and also get its response.
+
+Instantiate with:
+
+    require 'crash_watch/gdb_controller'
+    gdb = CrashWatch::GdbController.new
+
+This will spawn a new GDB process. Use `#execute` to execute arbitary GDB commands. Whatever the command prints to stdout and stderr will be available in the result string.
+
+    gdb.execute("bt")        # => backtrace string
+    gdb.execute("p 1 + 2")   # => "$1 = 3\n"
+
+Call `#close` when you no longer need it.
+
+    gdb.close

data/bin/crash-watch

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+require 'optparse'
+
+options = {}
+parser = OptionParser.new do |opts|
+	opts.banner = "Usage: crash-watch [options] PID"
+	opts.separator ""
+	
+	opts.separator "Options:"
+	opts.on("-d", "--debug", "Show GDB commands that crash-watch sends.") do
+		options[:debug] = true
+	end
+	opts.on("--dump", "Dump current process backtrace and exit immediately.") do
+		options[:dump] = true
+	end
+	opts.on("-v", "--version", "Show version number.") do
+		options[:version] = true
+	end
+	opts.on("-h", "--help", "Show this help message.") do
+		options[:help] = true
+	end
+end
+begin
+	parser.parse!
+rescue OptionParser::ParseError => e
+	puts e
+	puts
+	puts "Please see '--help' for valid options."
+	exit 1
+end
+
+if options[:help]
+	puts parser
+	exit
+elsif options[:version]
+	require 'crash_watch/version'
+	puts "crash-watch version #{CrashWatch::VERSION_STRING}"
+	exit
+elsif ARGV.size != 1
+	puts parser
+	exit 1
+end
+
+require 'crash_watch/gdb_controller'
+gdb = CrashWatch::GdbController.new
+begin
+	gdb.debug = options[:debug]
+	
+	# Ruby sometimes uses SIGVTARLM for thread scheduling.
+	gdb.execute("handle SIGVTALRM noprint pass")
+	
+	if gdb.attach(ARGV[0])
+		if options[:dump]
+			puts "Current thread (#{gdb.current_thread}) backtrace:"
+			puts "    " << gdb.current_thread_backtrace.gsub(/\n/, "\n    ")
+			puts
+			puts "All thread backtraces:"
+			puts "    " << gdb.all_threads_backtraces.gsub(/\n/, "\n    ")
+		else
+			puts "Monitoring PID #{ARGV[0]}..."
+			exit_info = gdb.wait_until_exit
+			puts "Process exited at #{Time.now}."
+			puts "Exit code: #{exit_info.exit_code}" if exit_info.exit_code
+			puts "Signal: #{exit_info.signal}" if exit_info.signaled?
+			if exit_info.backtrace
+				puts "Backtrace:"
+				puts "    " << exit_info.backtrace.gsub(/\n/, "\n    ")
+			end
+		end
+	else
+		puts "Cannot attach to process."
+		exit 2
+	end
+ensure
+	gdb.close
+end

data/crash-watch.gemspec

@@ -0,0 +1,26 @@
+require File.expand_path('lib/crash_watch/version', File.dirname(__FILE__))
+
+Gem::Specification.new do |s|
+	s.name = "crash-watch"
+	s.version = CrashWatch::VERSION_STRING
+	s.authors = ["Hongli Lai"]
+	s.date = "2010-04-16"
+	s.description = "Monitor processes and display useful information when they crash."
+	s.summary = "Monitor processes and display useful information when they crash"
+	s.email = "hongli@phusion.nl"
+	s.files = Dir[
+		"README.markdown",
+		"LICENSE.txt",
+		"crash-watch.gemspec",
+		"bin/**/*",
+		"lib/**/*",
+		"test/**/*"
+	]
+	s.homepage = "http://github.com/FooBarWidget/crash-watch"
+	s.rdoc_options = ["--charset=UTF-8"]
+	s.executables = ["crash-watch"]
+	s.require_paths = ["lib"]
+	s.add_development_dependency("ffi")
+	s.add_development_dependency("rspec")
+end
+

data/lib/crash_watch/gdb_controller.rb

@@ -0,0 +1,256 @@
+module CrashWatch
+
+class GdbController
+	class ExitInfo
+		attr_reader :exit_code, :signal, :backtrace, :snapshot
+		
+		def initialize(exit_code, signal, backtrace, snapshot)
+			@exit_code = exit_code
+			@signal = signal
+			@backtrace = backtrace
+			@snapshot = snapshot
+		end
+		
+		def signaled?
+			!!@signal
+		end
+	end
+	
+	END_OF_RESPONSE_MARKER = '--------END_OF_RESPONSE--------'
+	
+	attr_accessor :debug
+	
+	def initialize
+		@pid, @in, @out = popen_command("gdb", "-n", "-q")
+		execute("set prompt ")
+	end
+	
+	def execute(command_string, timeout = nil)
+		raise "GDB session is already closed" if !@pid
+		puts "gdb write #{command_string.inspect}" if @debug
+		@in.puts(command_string)
+		@in.puts("echo \\n#{END_OF_RESPONSE_MARKER}\\n")
+		done = false
+		result = ""
+		while !done
+			begin
+				if select([@out], nil, nil, timeout)
+					line = @out.readline
+					puts "gdb read #{line.inspect}" if @debug
+					if line == "#{END_OF_RESPONSE_MARKER}\n"
+						done = true
+					else
+						result << line
+					end
+				else
+					close!
+					done = true
+					result = nil
+				end
+			rescue EOFError
+				done = true
+			end
+		end
+		return result
+	end
+	
+	def closed?
+		return !@pid
+	end
+	
+	def close
+		if !closed?
+			begin
+				execute("detach", 5)
+				execute("quit", 5) if !closed?
+			rescue Errno::EPIPE
+			end
+			if !closed?
+				@in.close
+				@out.close
+				Process.waitpid(@pid)
+				@pid = nil
+			end
+		end
+	end
+	
+	def close!
+		if !closed?
+			@in.close
+			@out.close
+			Process.kill('KILL', @pid)
+			Process.waitpid(@pid)
+			@pid = nil
+		end
+	end
+	
+	def attach(pid)
+		pid = pid.to_s.strip
+		raise ArgumentError if pid.empty?
+		result = execute("attach #{pid}")
+		return result !~ /(No such process|Unable to access task|Operation not permitted)/
+	end
+	
+	def call(code)
+		result = execute("call #{code}")
+		result =~ /= (.*)$/
+		return $1
+	end
+	
+	def program_counter
+		return execute("p/x $pc").gsub(/.* = /, '')
+	end
+	
+	def current_thread
+		execute("thread") =~ /Current thread is (.+?) /
+		return $1
+	end
+	
+	def current_thread_backtrace
+		return execute("bt full").strip
+	end
+	
+	def all_threads_backtraces
+		return execute("thread apply all bt full").strip
+	end
+	
+	def ruby_backtrace
+		filename = "/tmp/gdb-capture.#{@pid}.txt"
+		
+		orig_stdout_fd_copy = call("(int) dup(1)")
+		new_stdout = call(%Q{(void *) fopen("#{filename}", "w")})
+		new_stdout_fd = call("(int) fileno(#{new_stdout})")
+		call("(int) dup2(#{new_stdout_fd}, 1)")
+		
+		# Let's hope stdout is set to line buffered or unbuffered mode...
+		call("(void) rb_backtrace()")
+		
+		call("(int) dup2(#{orig_stdout_fd_copy}, 1)")
+		call("(int) fclose(#{new_stdout})")
+		call("(int) close(#{orig_stdout_fd_copy})")
+		
+		if File.exist?(filename)
+			result = File.read(filename)
+			result.strip!
+			if result.empty?
+				return nil
+			else
+				return result
+			end
+		else
+			return nil
+		end
+	ensure
+		if filename
+			File.unlink(filename) rescue nil
+		end
+	end
+	
+	def wait_until_exit
+		execute("break _exit")
+		
+		signal = nil
+		backtraces = nil
+		snapshot = nil
+		
+		while true
+			result = execute("continue")
+			if result =~ /^Program received signal (.+?),/
+				signal = $1
+				backtraces = execute("thread apply all bt full").strip
+				if backtraces.empty?
+					backtraces = execute("bt full").strip
+				end
+				snapshot = yield(self) if block_given?
+				
+				# This signal may or may not be immediately fatal; the
+				# signal might be ignored by the process, or the process
+				# has some clever signal handler that fixes the state,
+				# or maybe the signal handler must run some cleanup code
+				# before killing the process. Let's find out by running
+				# the next machine instruction.
+				old_program_counter = program_counter
+				result = execute("stepi")
+				if result =~ /^Program received signal .+?,/
+					# Yes, it was fatal. Here we don't care whether the
+					# instruction caused a different signal. The last
+					# one is probably what we're interested in.
+					return ExitInfo.new(nil, signal, backtraces, snapshot)
+				elsif result =~ /^Program (terminated|exited)/ || result =~ /^Breakpoint .*? _exit/
+					# Running the next instruction causes the program to terminate.
+					# Not sure what's going on but the previous signal and
+					# backtrace is probably what we're interested in.
+					return ExitInfo.new(nil, signal, backtraces, snapshot)
+				elsif old_program_counter == program_counter
+					# The process cannot continue but we're not sure what GDB
+					# is telling us.
+					raise "Unexpected GDB output: #{result}"
+				end
+				# else:
+				# The signal doesn't isn't immediately fatal, so save current
+				# status, continue, and check whether the process exits later.
+			elsif result =~ /^Program terminated with signal (.+?),/
+				if $1 == signal
+					# Looks like the signal we trapped earlier
+					# caused an exit.
+					return ExitInfo.new(nil, signal, backtraces, snapshot)
+				else
+					return ExitInfo.new(nil, signal, nil, snapshot)
+				end
+			elsif result =~ /^Breakpoint .*? _exit /
+				backtraces = execute("thread apply all bt full").strip
+				if backtraces.empty?
+					backtraces = execute("bt full").strip
+				end
+				snapshot = yield(self) if block_given?
+				# On OS X, gdb may fail to return from the 'continue' command
+				# even though the process exited. Kernel bug? In any case,
+				# we put a timeout here so that we don't wait indefinitely.
+				result = execute("continue", 10)
+				if result =~ /^Program exited with code (\d+)\.$/
+					return ExitInfo.new($1.to_i, nil, backtraces, snapshot)
+				elsif result =~ /^Program exited normally/
+					return ExitInfo.new(0, nil, backtraces, snapshot)
+				else
+					return ExitInfo.new(nil, nil, backtraces, snapshot)
+				end
+			elsif result =~ /^Program exited with code (\d+)\.$/
+				return ExitInfo.new($1.to_i, nil, nil, nil)
+			elsif result =~ /^Program exited normally/
+				return ExitInfo.new(0, nil, nil, nil)
+			else
+				return ExitInfo.new(nil, nil, nil, nil)
+			end
+		end
+	end
+	
+private
+	def popen_command(*command)
+		a, b = IO.pipe
+		c, d = IO.pipe
+		if Process.respond_to?(:spawn)
+			args = command.dup
+			args << {
+				STDIN  => a,
+				STDOUT => d,
+				STDERR => d,
+				:close_others => true
+			}
+			pid = Process.spawn(*args)
+		else
+			pid = fork do
+				STDIN.reopen(a)
+				STDOUT.reopen(d)
+				STDERR.reopen(d)
+				b.close
+				c.close
+				exec(*command)
+			end
+		end
+		a.close
+		d.close
+		return [pid, b, c]
+	end
+end
+
+end

data/lib/crash_watch/version.rb

@@ -0,0 +1,3 @@
+module CrashWatch
+	VERSION_STRING = '1.1.0'
+end

data/test/gdb_controller_spec.rb

@@ -0,0 +1,116 @@
+source_root = File.expand_path(File.dirname(__FILE__) + "/..")
+$LOAD_PATH.unshift("#{source_root}/lib")
+Thread.abort_on_exception = true
+
+require 'crash_watch/gdb_controller'
+
+describe CrashWatch::GdbController do
+	before :each do
+		@gdb = CrashWatch::GdbController.new
+	end
+	
+	after :each do
+		@gdb.close
+		if @process
+			Process.kill('KILL', @process.pid)
+			@process.close
+		end
+	end
+	
+	def run_script_and_wait(code, snapshot_callback = nil, &block)
+		@process = IO.popen(%Q{ruby -e '#{code}'}, 'w')
+		@gdb.attach(@process.pid)
+		thread = Thread.new do
+			sleep 0.1
+			if block
+				block.call
+			end
+			@process.write("\n")
+		end
+		exit_info = @gdb.wait_until_exit(&snapshot_callback)
+		thread.join
+		return exit_info
+	end
+	
+	describe "#execute" do
+		it "executes the desired command and returns its output" do
+			@gdb.execute("echo hello world").should == "hello world\n"
+		end
+	end
+	
+	describe "#attach" do
+		before :each do
+			@process = IO.popen("sleep 9999", "w")
+		end
+		
+		it "returns true if attaching worked" do
+			@gdb.attach(@process.pid).should be_true
+		end
+		
+		it "returns false if the PID doesn't exist" do
+			Process.kill('KILL', @process.pid)
+			sleep 0.25
+			@gdb.attach(@process.pid).should be_false
+		end
+	end
+	
+	describe "#wait_until_exit" do
+		it "returns the expected information if the process exited normally" do
+			exit_info = run_script_and_wait('STDIN.readline')
+			exit_info.exit_code.should == 0
+			exit_info.should_not be_signaled
+		end
+		
+		it "returns the expected information if the process exited with a non-zero exit code" do
+			exit_info = run_script_and_wait('STDIN.readline; exit 3')
+			exit_info.exit_code.should == 3
+			exit_info.should_not be_signaled
+			exit_info.backtrace.should_not be_nil
+			exit_info.backtrace.should_not be_empty
+		end
+		
+		it "returns the expected information if the process exited because of a signal" do
+			exit_info = run_script_and_wait(
+				'STDIN.readline;' +
+				'require "rubygems";' +
+				'require "ffi";' +
+				'module MyLib;' +
+					'extend FFI::Library;' +
+					'ffi_lib "c";' +
+					'attach_function :abort, [], :void;' +
+				'end;' +
+				'MyLib.abort')
+			exit_info.should be_signaled
+			exit_info.backtrace.should =~ /abort/
+		end
+		
+		it "ignores non-fatal signals" do
+			exit_info = run_script_and_wait('trap("INT") { }; STDIN.readline; exit 2') do
+				Process.kill('INT', @process.pid)
+			end
+			exit_info.exit_code.should == 2
+			exit_info.should_not be_signaled
+			exit_info.backtrace.should_not be_nil
+			exit_info.backtrace.should_not be_empty
+		end
+		
+		it "returns information of the signal that aborted the process, not information of ignored signals" do
+			exit_info = run_script_and_wait(
+				'trap("INT") { };' +
+				'STDIN.readline;' +
+				'require "rubygems";' +
+				'require "ffi";' +
+				'module MyLib;' +
+					'extend FFI::Library;' +
+					'ffi_lib "c";' +
+					'attach_function :abort, [], :void;' +
+				'end;' +
+				'MyLib.abort'
+			) do
+				Process.kill('INT', @process.pid)
+			end
+			exit_info.should be_signaled
+			exit_info.backtrace.should =~ /abort/
+		end
+	end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification 
+name: crash-watch
+version: !ruby/object:Gem::Version 
+  hash: 19
+  prerelease: false
+  segments: 
+  - 1
+  - 1
+  - 0
+  version: 1.1.0
+platform: ruby
+authors: 
+- Hongli Lai
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-04-16 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: ffi
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+description: Monitor processes and display useful information when they crash.
+email: hongli@phusion.nl
+executables: 
+- crash-watch
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.markdown
+- LICENSE.txt
+- crash-watch.gemspec
+- bin/crash-watch
+- lib/crash_watch/gdb_controller.rb
+- lib/crash_watch/version.rb
+- test/gdb_controller_spec.rb
+has_rdoc: true
+homepage: http://github.com/FooBarWidget/crash-watch
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Monitor processes and display useful information when they crash
+test_files: []
+