bluepill-rwgps 0.0.60

data/lib/bluepill/process_conditions.rb

@@ -0,0 +1,14 @@
+# -*- encoding: utf-8 -*-
+module Bluepill
+  module ProcessConditions
+    def self.[](name)
+      const_get(name.to_s.camelcase)
+    end
+  end
+end
+
+require "bluepill/process_conditions/process_condition"
+Dir["#{File.dirname(__FILE__)}/process_conditions/*.rb"].each do |pc|
+  require pc
+end
+

data/lib/bluepill/process_statistics.rb

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+module Bluepill
+  class ProcessStatistics
+    STRFTIME = "%m/%d/%Y %H:%I:%S".freeze
+    EVENTS_TO_PERSIST = 10
+
+    attr_reader :events
+
+    # possibly persist this data.
+    def initialize
+      @events = Util::RotationalArray.new(EVENTS_TO_PERSIST)
+    end
+
+    def record_event(event, reason)
+      events.push([event, reason, Time.now])
+    end
+
+    def to_s
+      str = events.reverse.map do |(event, reason, time)|
+        "  #{event} at #{time.strftime(STRFTIME)} - #{reason || "unspecified"}"
+      end.join("\n")
+
+      "event history:\n#{str}"
+    end
+  end
+end
+

data/lib/bluepill/socket.rb

@@ -0,0 +1,58 @@
+# -*- encoding: utf-8 -*-
+require 'socket'
+
+module Bluepill
+  module Socket
+    TIMEOUT = 60 # Used for client commands
+    MAX_ATTEMPTS = 5
+
+    extend self
+
+    def client(base_dir, name, &block)
+      UNIXSocket.open(socket_path(base_dir, name), &block)
+    end
+
+    def client_command(base_dir, name, command)
+      res = nil
+      MAX_ATTEMPTS.times do |current_attempt|
+        begin
+          client(base_dir, name) do |socket|
+            Timeout.timeout(TIMEOUT) do
+              socket.puts command
+              res = Marshal.load(socket.read)
+            end
+          end
+          break
+        rescue EOFError, Timeout::Error
+          if current_attempt == MAX_ATTEMPTS - 1
+            abort("Socket Timeout: Server may not be responding")
+          end
+          puts "Retry #{current_attempt + 1} of #{MAX_ATTEMPTS}"
+        end
+      end
+      res
+    end
+
+    def server(base_dir, name)
+      socket_path = self.socket_path(base_dir, name)
+      begin
+        UNIXServer.open(socket_path)
+      rescue Errno::EADDRINUSE
+        # if sock file has been created.  test to see if there is a server
+        begin
+          UNIXSocket.open(socket_path)
+        rescue Errno::ECONNREFUSED
+          File.delete(socket_path)
+          return UNIXServer.open(socket_path)
+        else
+          logger.err("Server is already running!")
+          exit(7)
+        end
+      end
+    end
+
+    def socket_path(base_dir, name)
+      File.join(base_dir, 'socks', name + ".sock")
+    end
+  end
+end

data/lib/bluepill/system.rb

@@ -0,0 +1,238 @@
+# -*- encoding: utf-8 -*-
+require 'etc'
+require "shellwords"
+
+module Bluepill
+  # This class represents the system that bluepill is running on.. It's mainly used to memoize
+  # results of running ps auxx etc so that every watch in the every process will not result in a fork
+  module System
+    APPEND_MODE = "a"
+    extend self
+
+    # The position of each field in ps output
+    IDX_MAP = {
+      :pid => 0,
+      :ppid => 1,
+      :pcpu => 2,
+      :rss => 3
+    }
+
+    def pid_alive?(pid)
+      begin
+        ::Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+
+    def cpu_usage(pid)
+      ps_axu[pid] && ps_axu[pid][IDX_MAP[:pcpu]].to_f
+    end
+
+    def memory_usage(pid)
+      ps_axu[pid] && ps_axu[pid][IDX_MAP[:rss]].to_f
+    end
+
+    def get_children(parent_pid)
+      child_pids = Array.new
+      ps_axu.each_pair do |pid, chunks|
+        child_pids << chunks[IDX_MAP[:pid]].to_i if chunks[IDX_MAP[:ppid]].to_i == parent_pid.to_i
+      end
+      child_pids
+    end
+
+    # Returns the pid of the child that executes the cmd
+    def daemonize(cmd, options = {})
+      rd, wr = IO.pipe
+
+      if child = Daemonize.safefork
+        # we do not wanna create zombies, so detach ourselves from the child exit status
+        ::Process.detach(child)
+
+        # parent
+        wr.close
+
+        daemon_id = rd.read.to_i
+        rd.close
+
+        return daemon_id if daemon_id > 0
+
+      else
+        # child
+        rd.close
+
+        drop_privileges(options[:uid], options[:gid], options[:supplementary_groups])
+
+        # if we cannot write the pid file as the provided user, err out
+        exit unless can_write_pid_file(options[:pid_file], options[:logger])
+
+        to_daemonize = lambda do
+          # Setting end PWD env emulates bash behavior when dealing with symlinks
+          Dir.chdir(ENV["PWD"] = options[:working_dir].to_s)  if options[:working_dir]
+          options[:environment].each { |key, value| ENV[key.to_s] = value.to_s } if options[:environment]
+
+          redirect_io(*options.values_at(:stdin, :stdout, :stderr))
+
+          ::Kernel.exec(*Shellwords.shellwords(cmd))
+          exit
+        end
+
+        daemon_id = Daemonize.call_as_daemon(to_daemonize, nil, cmd)
+
+        File.open(options[:pid_file], "w") {|f| f.write(daemon_id)}
+
+        wr.write daemon_id
+        wr.close
+
+        exit
+      end
+    end
+
+    # Returns the stdout, stderr and exit code of the cmd
+    def execute_blocking(cmd, options = {})
+      rd, wr = IO.pipe
+
+      if child = Daemonize.safefork
+        # parent
+        wr.close
+
+        cmd_status = rd.read
+        rd.close
+
+        ::Process.waitpid(child)
+
+        cmd_status.strip != '' ? Marshal.load(cmd_status) : {:exit_code => 0, :stdout => '', :stderr => ''}
+      else
+        # child
+        rd.close
+
+        # create a child in which we can override the stdin, stdout and stderr
+        cmd_out_read, cmd_out_write = IO.pipe
+        cmd_err_read, cmd_err_write = IO.pipe
+
+        pid = fork {
+          # grandchild
+          drop_privileges(options[:uid], options[:gid], options[:supplementary_groups])
+
+          Dir.chdir(ENV["PWD"] = options[:working_dir].to_s) if options[:working_dir]
+          options[:environment].each { |key, value| ENV[key.to_s] = value.to_s } if options[:environment]
+
+          # close unused fds so ancestors wont hang. This line is the only reason we are not
+          # using something like popen3. If this fd is not closed, the .read call on the parent
+          # will never return because "wr" would still be open in the "exec"-ed cmd
+          wr.close
+
+          # we do not care about stdin of cmd
+          STDIN.reopen("/dev/null")
+
+          # point stdout of cmd to somewhere we can read
+          cmd_out_read.close
+          STDOUT.reopen(cmd_out_write)
+          cmd_out_write.close
+
+          # same thing for stderr
+          cmd_err_read.close
+          STDERR.reopen(cmd_err_write)
+          cmd_err_write.close
+
+          # finally, replace grandchild with cmd
+          ::Kernel.exec(*Shellwords.shellwords(cmd))
+        }
+
+        # we do not use these ends of the pipes in the child
+        cmd_out_write.close
+        cmd_err_write.close
+
+        # wait for the cmd to finish executing and acknowledge it's death
+        ::Process.waitpid(pid)
+
+        # collect stdout, stderr and exitcode
+        result = {
+          :stdout => cmd_out_read.read,
+          :stderr => cmd_err_read.read,
+          :exit_code => $?.exitstatus
+        }
+
+        # We're done with these ends of the pipes as well
+        cmd_out_read.close
+        cmd_err_read.close
+
+        # Time to tell the parent about what went down
+        wr.write Marshal.dump(result)
+        wr.close
+
+        ::Process.exit!
+      end
+    end
+
+    def store
+      @store ||= Hash.new
+    end
+
+    def reset_data
+      store.clear unless store.empty?
+    end
+
+    def ps_axu
+      # TODO: need a mutex here
+      store[:ps_axu] ||= begin
+        # BSD style ps invocation
+        lines = `ps axo pid,ppid,pcpu,rss`.split("\n")
+
+        lines.inject(Hash.new) do |mem, line|
+          chunks = line.split(/\s+/)
+          chunks.delete_if {|c| c.strip.empty? }
+          pid = chunks[IDX_MAP[:pid]].strip.to_i
+          mem[pid] = chunks
+          mem
+        end
+      end
+    end
+
+    # be sure to call this from a fork otherwise it will modify the attributes
+    # of the bluepill daemon
+    def drop_privileges(uid, gid, supplementary_groups)
+      if ::Process::Sys.geteuid == 0
+        uid_num = Etc.getpwnam(uid).uid if uid
+        gid_num = Etc.getgrnam(gid).gid if gid
+
+        supplementary_groups ||= []
+
+        group_nums = supplementary_groups.map do |group|
+          Etc.getgrnam(group).gid
+        end
+
+        ::Process.groups = [gid_num] if gid
+        ::Process.groups |= group_nums unless group_nums.empty?
+        ::Process::Sys.setgid(gid_num) if gid
+        ::Process::Sys.setuid(uid_num) if uid
+        ENV['HOME'] = Etc.getpwuid(uid_num).try(:dir) || ENV['HOME'] if uid
+      end
+    end
+
+    def can_write_pid_file(pid_file, logger)
+      FileUtils.touch(pid_file)
+      File.unlink(pid_file)
+      return true
+
+    rescue Exception => e
+      logger.warning "%s - %s" % [e.class.name, e.message]
+      e.backtrace.each {|l| logger.warning l}
+      return false
+    end
+
+    def redirect_io(io_in, io_out, io_err)
+      $stdin.reopen(io_in) if io_in
+
+      if !io_out.nil? && !io_err.nil? && io_out == io_err
+        $stdout.reopen(io_out, APPEND_MODE)
+        $stderr.reopen($stdout)
+
+      else
+        $stdout.reopen(io_out, APPEND_MODE) if io_out
+        $stderr.reopen(io_err, APPEND_MODE) if io_err
+      end
+    end
+  end
+end

data/lib/bluepill/trigger.rb

@@ -0,0 +1,60 @@
+# -*- encoding: utf-8 -*-
+module Bluepill
+  class Trigger
+    @implementations = {}
+    def self.inherited(klass)
+      @implementations[klass.name.split('::').last.underscore.to_sym] = klass
+    end
+
+    def self.[](name)
+      @implementations[name]
+    end
+
+    attr_accessor :process, :logger, :mutex, :scheduled_events
+
+    def initialize(process, options = {})
+      self.process = process
+      self.logger = options[:logger]
+      self.mutex = Mutex.new
+      self.scheduled_events = []
+    end
+
+    def reset!
+      self.cancel_all_events
+    end
+
+    def notify(transition)
+      raise "Implement in subclass"
+    end
+
+    def dispatch!(event)
+      self.process.dispatch!(event, self.class.name.split("::").last)
+    end
+
+    def schedule_event(event, delay)
+      # TODO: maybe wrap this in a ScheduledEvent class with methods like cancel
+      thread = Thread.new(self) do |trigger|
+        begin
+          sleep delay.to_f
+          trigger.dispatch!(event)
+          trigger.mutex.synchronize do
+            trigger.scheduled_events.delete_if { |_, thread| thread == Thread.current }
+          end
+        rescue StandardError => e
+          trigger.logger.err(e)
+          trigger.logger.err(e.backtrace.join("\n"))
+        end
+      end
+
+      self.scheduled_events.push([event, thread])
+    end
+
+    def cancel_all_events
+      self.logger.info "Canceling all scheduled events"
+      self.mutex.synchronize do
+        self.scheduled_events.each {|_, thread| thread.kill}
+      end
+    end
+
+  end
+end

data/lib/bluepill/triggers/flapping.rb

@@ -0,0 +1,56 @@
+# -*- encoding: utf-8 -*-
+module Bluepill
+  module Triggers
+    class Flapping < Bluepill::Trigger
+      TRIGGER_STATES = [:starting, :restarting]
+
+      PARAMS = [:times, :within, :retry_in]
+
+      attr_accessor *PARAMS
+      attr_reader :timeline
+
+      def initialize(process, options = {})
+        options.reverse_merge!(:times => 5, :within => 1, :retry_in => 5)
+
+        options.each_pair do |name, val|
+          instance_variable_set("@#{name}", val) if PARAMS.include?(name)
+        end
+
+        @timeline = Util::RotationalArray.new(@times)
+        super
+      end
+
+      def notify(transition)
+        if TRIGGER_STATES.include?(transition.to_name)
+          self.timeline << Time.now.to_i
+          self.check_flapping
+        end
+      end
+
+      def reset!
+        @timeline.clear
+        super
+      end
+
+      def check_flapping
+        # The process has not flapped if we haven't encountered enough incidents
+        return unless (@timeline.compact.length == self.times)
+        
+        # Check if the incident happend within the timeframe
+        duration = (@timeline.last - @timeline.first) <= self.within
+
+        if duration
+          self.logger.info "Flapping detected: retrying in #{self.retry_in} seconds"
+
+          self.schedule_event(:start, self.retry_in) unless self.retry_in == 0 # retry_in zero means "do not retry, ever"
+          self.schedule_event(:unmonitor, 0)
+
+          @timeline.clear
+
+          # This will prevent a transition from happening in the process state_machine
+          throw :halt
+        end
+      end
+    end
+  end
+end

data/lib/bluepill/util/rotational_array.rb

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+module Bluepill
+  module Util
+    class RotationalArray < Array
+      def initialize(size)
+        @capacity = size
+
+        super() # no size - intentionally
+      end
+
+      def push(value)
+        super(value)
+
+        self.shift if self.length > @capacity
+        self
+      end
+      alias_method :<<, :push
+    end
+  end
+end

data/lib/bluepill/version.rb

@@ -0,0 +1,4 @@
+# -*- encoding: utf-8 -*-
+module Bluepill
+  VERSION = "0.0.60".freeze
+end

data/lib/bluepill.rb

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+require 'rubygems'
+
+require 'bundler/setup' if ENV['BUNDLE_GEMFILE'] && File.exists?(ENV['BUNDLE_GEMFILE'])
+
+require 'thread'
+require 'monitor'
+require 'syslog'
+require 'timeout'
+require 'logger'
+
+require 'active_support/inflector'
+require 'active_support/core_ext/hash'
+require 'active_support/core_ext/numeric'
+require 'active_support/duration'
+
+require 'bluepill/dsl/process_proxy'
+require 'bluepill/dsl/process_factory'
+require 'bluepill/dsl/app_proxy'
+
+require 'bluepill/application'
+require 'bluepill/controller'
+require 'bluepill/socket'
+require "bluepill/process"
+require "bluepill/process_statistics"
+require "bluepill/group"
+require "bluepill/logger"
+require "bluepill/condition_watch"
+require 'bluepill/trigger'
+require 'bluepill/triggers/flapping'
+require "bluepill/dsl"
+require "bluepill/system"
+
+require "bluepill/process_conditions"
+
+require "bluepill/util/rotational_array"
+
+require "bluepill/version"

data/local-bluepill

@@ -0,0 +1,129 @@
+#! /usr/bin/env ruby
+
+require 'rubygems'
+require 'bundler/setup'
+$LOAD_PATH.unshift(File.expand_path('lib', File.dirname(__FILE__)))
+
+require 'optparse'
+require 'bluepill'
+
+begin
+  require 'rbconfig'
+rescue LoadError
+end
+
+RbConfig = Config unless Object.const_defined?(:RbConfig)
+
+# Default options
+options = {
+  :log_file   => "/var/log/bluepill.log",
+  :base_dir   => "/var/run/bluepill",
+  :privileged => true,
+  :timeout    => 10,
+  :attempts   => 1
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: bluepill [app] cmd [options]"
+  opts.on('-l', "--logfile LOGFILE", "Path to logfile, defaults to #{options[:log_file]}") do |file|
+    options[:log_file] = file
+  end
+
+  opts.on('-c', "--base-dir DIR", "Directory to store bluepill socket and pid files, defaults to #{options[:base_dir]}") do |base_dir|
+    options[:base_dir] = base_dir
+  end
+
+  opts.on("-v", "--version") do
+    puts "bluepill, version #{Bluepill::VERSION}"
+    exit
+  end
+
+  opts.on("--[no-]privileged", "Allow/disallow to run #{$0} as non-privileged process. disallowed by default") do |v|
+    options[:privileged] = v
+  end
+
+  opts.on('-t', '--timeout Seconds', Integer, "Timeout for commands sent to the daemon, in seconds. Defaults to 10.") do |timeout|
+    options[:timeout] = timeout
+  end
+
+  opts.on('--attempts Count', Integer, "Attempts for commands sent to the daemon, in seconds. Defaults to 1.") do |attempts|
+    options[:attempts] = attempts
+  end
+
+  help = proc do
+    puts opts
+    puts
+    puts "Commands:"
+    puts "    load CONFIG_FILE\t\tLoads new instance of bluepill using the specified config file"
+    puts "    status\t\t\tLists the status of the proceses for the specified app"
+    puts "    start [TARGET]\t\tIssues the start command for the target process or group, defaults to all processes"
+    puts "    stop [TARGET]\t\tIssues the stop command for the target process or group, defaults to all processes"
+    puts "    restart [TARGET]\t\tIssues the restart command for the target process or group, defaults to all processes"
+    puts "    unmonitor [TARGET]\t\tStop monitoring target process or group, defaults to all processes"
+    puts "    log [TARGET]\t\tShow the log for the specified process or group, defaults to all for app"
+    puts "    quit\t\t\tStop bluepill"
+    puts
+    puts "See http://github.com/arya/bluepill for README"
+    exit
+  end
+
+  opts.on_tail('-h','--help', 'Show this message', &help)
+  help.call if ARGV.empty?
+end.parse!
+
+# Check for root
+if options[:privileged] && ::Process.euid != 0
+  $stderr.puts "You must run bluepill as root or use --no-privileged option."
+  exit(3)
+end
+
+APPLICATION_COMMANDS = %w(status start stop restart unmonitor quit log)
+
+controller = Bluepill::Controller.new(options.slice(:base_dir, :log_file))
+
+if controller.running_applications.include?(File.basename($0)) && File.symlink?($0)
+  # bluepill was called as a symlink with the name of the target application
+  options[:application] = File.basename($0)
+elsif controller.running_applications.include?(ARGV.first)
+  # the first arg is the application name
+  options[:application] = ARGV.shift
+elsif APPLICATION_COMMANDS.include?(ARGV.first)
+  if controller.running_applications.length == 1
+    # there is only one, let's just use that
+    options[:application] = controller.running_applications.first
+  elsif controller.running_applications.length > 1
+    # There is more than one, tell them the list and exit
+    $stderr.puts "You must specify an application name to run that command. Here's the list of running applications:"
+    controller.running_applications.each_with_index do |app, index|
+      $stderr.puts "  #{index + 1}. #{app}"
+    end
+    $stderr.puts "Usage: bluepill [app] cmd [options]"
+    exit(1)
+  else
+    # There are none running AND they aren't trying to start one
+    $stderr.puts "Error: There are no running bluepill daemons.\nTo start a bluepill daemon, use: bluepill load <config file>"
+    exit(2)
+  end
+end
+
+options[:command] = ARGV.shift
+
+if options[:command] == "load"
+  file = ARGV.shift
+  if File.exists?(file)
+    # Restart the ruby interpreter for the config file so that anything loaded here
+    # does not stay in memory for the daemon
+    ruby = File.join(RbConfig::CONFIG['bindir'], RbConfig::CONFIG['ruby_install_name'])
+    load_path = File.expand_path("#{File.dirname(__FILE__)}/../lib")
+    file_path = File.expand_path(file)
+
+    exec(ruby, "-I#{load_path}", '-rbluepill', file_path)
+    
+  else
+    $stderr.puts "Can't find file: #{file}"
+  end
+else
+  target = ARGV.shift
+  controller.handle_command(options[:application], options[:command], target)
+end
+

data/spec/lib/bluepill/logger_spec.rb

@@ -0,0 +1,3 @@
+describe Bluepill::Logger do
+
+end

data/spec/lib/bluepill/process_statistics_spec.rb

@@ -0,0 +1,24 @@
+describe Bluepill::ProcessStatistics do
+  before(:each) do
+    @stats = Bluepill::ProcessStatistics.new
+  end
+
+  it "should record events" do
+    @stats.record_event('some event', 'some reason')
+    @stats.record_event('another event', 'another reason')
+    @stats.events.should have(2).events
+  end
+
+  it "should record #EVENTS_TO_PERSIST events" do
+    (2 * Bluepill::ProcessStatistics::EVENTS_TO_PERSIST).times do
+      @stats.record_event('some event', 'some reason')
+    end
+    @stats.events.should have(Bluepill::ProcessStatistics::EVENTS_TO_PERSIST).events
+  end
+
+  it "should return event history" do
+    @stats.record_event('some event', 'some reason')
+    @stats.to_s.should match(/some reason/)
+    @stats.to_s.should match(/event history/)
+  end
+end