cloud66-bluepill 0.0.62

data/lib/bluepill/socket.rb

@@ -0,0 +1,58 @@
+# -*- encoding: utf-8 -*-
+require 'socket'
+
+module Bluepill
+  module Socket
+    TIMEOUT = 60 # Used for client commands
+    MAX_ATTEMPTS = 5
+
+    extend self
+
+    def client(base_dir, name, &block)
+      UNIXSocket.open(socket_path(base_dir, name), &block)
+    end
+
+    def client_command(base_dir, name, command)
+      res = nil
+      MAX_ATTEMPTS.times do |current_attempt|
+        begin
+          client(base_dir, name) do |socket|
+            Timeout.timeout(TIMEOUT) do
+              socket.puts command
+              res = Marshal.load(socket.read)
+            end
+          end
+          break
+        rescue EOFError, Timeout::Error
+          if current_attempt == MAX_ATTEMPTS - 1
+            abort("Socket Timeout: Server may not be responding")
+          end
+          puts "Retry #{current_attempt + 1} of #{MAX_ATTEMPTS}"
+        end
+      end
+      res
+    end
+
+    def server(base_dir, name)
+      socket_path = self.socket_path(base_dir, name)
+      begin
+        UNIXServer.open(socket_path)
+      rescue Errno::EADDRINUSE
+        # if sock file has been created.  test to see if there is a server
+        begin
+          UNIXSocket.open(socket_path)
+        rescue Errno::ECONNREFUSED
+          File.delete(socket_path)
+          return UNIXServer.open(socket_path)
+        else
+          logger.err("Server is already running!")
+          exit(7)
+        end
+      end
+    end
+
+    def socket_path(base_dir, name)
+      File.join(base_dir, 'socks', name + ".sock")
+    end
+  end
+end

data/lib/bluepill/system.rb

@@ -0,0 +1,265 @@
+# -*- encoding: utf-8 -*-
+require 'etc'
+require "shellwords"
+
+module Bluepill
+  # This class represents the system that bluepill is running on.. It's mainly used to memoize
+  # results of running ps auxx etc so that every watch in the every process will not result in a fork
+  module System
+    APPEND_MODE = "a"
+    extend self
+
+    # The position of each field in ps output
+    IDX_MAP = {
+      :pid => 0,
+      :ppid => 1,
+      :pcpu => 2,
+      :rss => 3
+    }
+
+    def pid_alive?(pid)
+      begin
+        ::Process.kill(0, pid)
+        true
+      rescue Errno::EPERM # no permission, but it is definitely alive
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+
+    def cpu_usage(pid, include_children)
+      ps = ps_axu
+      return unless ps[pid]
+      cpu_used = ps[pid][IDX_MAP[:pcpu]].to_f
+      get_children(pid).each { |child_pid|
+        cpu_used += ps[child_pid][IDX_MAP[:pcpu]].to_f if ps[child_pid]
+      } if include_children
+      cpu_used 
+    end
+
+    def memory_usage(pid, include_children)
+      ps = ps_axu
+      return unless ps[pid]
+      mem_used = ps[pid][IDX_MAP[:rss]].to_f
+      get_children(pid).each { |child_pid|
+        mem_used += ps[child_pid][IDX_MAP[:rss]].to_f if ps[child_pid]
+      } if include_children
+      mem_used
+    end
+
+    def get_children(parent_pid)
+      child_pids = Array.new
+      ps_axu.each_pair do |pid, chunks|
+        child_pids << chunks[IDX_MAP[:pid]].to_i if chunks[IDX_MAP[:ppid]].to_i == parent_pid.to_i
+      end
+      grand_children = child_pids.map{|pid| get_children(pid)}.flatten
+      child_pids.concat grand_children 
+    end
+
+    # Returns the pid of the child that executes the cmd
+    def daemonize(cmd, options = {})
+      rd, wr = IO.pipe
+
+      if child = Daemonize.safefork
+        # we do not wanna create zombies, so detach ourselves from the child exit status
+        ::Process.detach(child)
+
+        # parent
+        wr.close
+
+        daemon_id = rd.read.to_i
+        rd.close
+
+        return daemon_id if daemon_id > 0
+
+      else
+        # child
+        rd.close
+
+        drop_privileges(options[:uid], options[:gid], options[:supplementary_groups])
+
+        # if we cannot write the pid file as the provided user, err out
+        exit unless can_write_pid_file(options[:pid_file], options[:logger])
+
+        to_daemonize = lambda do
+          # Setting end PWD env emulates bash behavior when dealing with symlinks
+          Dir.chdir(ENV["PWD"] = options[:working_dir].to_s)  if options[:working_dir]
+          options[:environment].each { |key, value| ENV[key.to_s] = value.to_s } if options[:environment]
+
+          redirect_io(*options.values_at(:stdin, :stdout, :stderr))
+
+          ::Kernel.exec(*Shellwords.shellwords(cmd))
+          exit
+        end
+
+        daemon_id = Daemonize.call_as_daemon(to_daemonize, nil, cmd)
+
+        File.open(options[:pid_file], "w") {|f| f.write(daemon_id)}
+
+        wr.write daemon_id
+        wr.close
+
+        exit
+      end
+    end
+
+    def delete_if_exists(filename)
+      tries = 0
+
+      begin
+        File.unlink(filename) if filename && File.exists?(filename)
+      rescue IOError, Errno::ENOENT
+      rescue Errno::EACCES
+        retry if (tries += 1) < 3 
+        $stderr.puts("Warning: permission denied trying to delete #{filename}")
+      end
+    end
+
+    # Returns the stdout, stderr and exit code of the cmd
+    def execute_blocking(cmd, options = {})
+      rd, wr = IO.pipe
+
+      if child = Daemonize.safefork
+        # parent
+        wr.close
+
+        cmd_status = rd.read
+        rd.close
+
+        ::Process.waitpid(child)
+
+        cmd_status.strip != '' ? Marshal.load(cmd_status) : {:exit_code => 0, :stdout => '', :stderr => ''}
+      else
+        # child
+        rd.close
+
+        # create a child in which we can override the stdin, stdout and stderr
+        cmd_out_read, cmd_out_write = IO.pipe
+        cmd_err_read, cmd_err_write = IO.pipe
+
+        pid = fork {
+          # grandchild
+          drop_privileges(options[:uid], options[:gid], options[:supplementary_groups])
+
+          Dir.chdir(ENV["PWD"] = options[:working_dir].to_s) if options[:working_dir]
+          options[:environment].each { |key, value| ENV[key.to_s] = value.to_s } if options[:environment]
+
+          # close unused fds so ancestors wont hang. This line is the only reason we are not
+          # using something like popen3. If this fd is not closed, the .read call on the parent
+          # will never return because "wr" would still be open in the "exec"-ed cmd
+          wr.close
+
+          # we do not care about stdin of cmd
+          STDIN.reopen("/dev/null")
+
+          # point stdout of cmd to somewhere we can read
+          cmd_out_read.close
+          STDOUT.reopen(cmd_out_write)
+          cmd_out_write.close
+
+          # same thing for stderr
+          cmd_err_read.close
+          STDERR.reopen(cmd_err_write)
+          cmd_err_write.close
+
+          # finally, replace grandchild with cmd
+          ::Kernel.exec(*Shellwords.shellwords(cmd))
+        }
+
+        # we do not use these ends of the pipes in the child
+        cmd_out_write.close
+        cmd_err_write.close
+
+        # wait for the cmd to finish executing and acknowledge it's death
+        ::Process.waitpid(pid)
+
+        # collect stdout, stderr and exitcode
+        result = {
+          :stdout => cmd_out_read.read,
+          :stderr => cmd_err_read.read,
+          :exit_code => $?.exitstatus
+        }
+
+        # We're done with these ends of the pipes as well
+        cmd_out_read.close
+        cmd_err_read.close
+
+        # Time to tell the parent about what went down
+        wr.write Marshal.dump(result)
+        wr.close
+
+        exit
+      end
+    end
+
+    def store
+      @store ||= Hash.new
+    end
+
+    def reset_data
+      store.clear unless store.empty?
+    end
+
+    def ps_axu
+      # TODO: need a mutex here
+      store[:ps_axu] ||= begin
+        # BSD style ps invocation
+        lines = `ps axo pid,ppid,pcpu,rss`.split("\n")
+
+        lines.inject(Hash.new) do |mem, line|
+          chunks = line.split(/\s+/)
+          chunks.delete_if {|c| c.strip.empty? }
+          pid = chunks[IDX_MAP[:pid]].strip.to_i
+          mem[pid] = chunks
+          mem
+        end
+      end
+    end
+
+    # be sure to call this from a fork otherwise it will modify the attributes
+    # of the bluepill daemon
+    def drop_privileges(uid, gid, supplementary_groups)
+      if ::Process::Sys.geteuid == 0
+        uid_num = Etc.getpwnam(uid).uid if uid
+        gid_num = Etc.getgrnam(gid).gid if gid
+
+        supplementary_groups ||= []
+
+        group_nums = supplementary_groups.map do |group|
+          Etc.getgrnam(group).gid
+        end
+
+        ::Process.groups = [gid_num] if gid
+        ::Process.groups |= group_nums unless group_nums.empty?
+        ::Process::Sys.setgid(gid_num) if gid
+        ::Process::Sys.setuid(uid_num) if uid
+        ENV['HOME'] = Etc.getpwuid(uid_num).try(:dir) || ENV['HOME'] if uid
+      end
+    end
+
+    def can_write_pid_file(pid_file, logger)
+      FileUtils.touch(pid_file)
+      File.unlink(pid_file)
+      return true
+
+    rescue Exception => e
+      logger.warning "%s - %s" % [e.class.name, e.message]
+      e.backtrace.each {|l| logger.warning l}
+      return false
+    end
+
+    def redirect_io(io_in, io_out, io_err)
+      $stdin.reopen(io_in) if io_in
+
+      if !io_out.nil? && !io_err.nil? && io_out == io_err
+        $stdout.reopen(io_out, APPEND_MODE)
+        $stderr.reopen($stdout)
+
+      else
+        $stdout.reopen(io_out, APPEND_MODE) if io_out
+        $stderr.reopen(io_err, APPEND_MODE) if io_err
+      end
+    end
+  end
+end

data/lib/bluepill/trigger.rb

@@ -0,0 +1,60 @@
+# -*- encoding: utf-8 -*-
+module Bluepill
+  class Trigger
+    @implementations = {}
+    def self.inherited(klass)
+      @implementations[klass.name.split('::').last.underscore.to_sym] = klass
+    end
+
+    def self.[](name)
+      @implementations[name]
+    end
+
+    attr_accessor :process, :logger, :mutex, :scheduled_events
+
+    def initialize(process, options = {})
+      self.process = process
+      self.logger = options[:logger]
+      self.mutex = Mutex.new
+      self.scheduled_events = []
+    end
+
+    def reset!
+      self.cancel_all_events
+    end
+
+    def notify(transition)
+      raise "Implement in subclass"
+    end
+
+    def dispatch!(event)
+      self.process.dispatch!(event, self.class.name.split("::").last)
+    end
+
+    def schedule_event(event, delay)
+      # TODO: maybe wrap this in a ScheduledEvent class with methods like cancel
+      thread = Thread.new(self) do |trigger|
+        begin
+          sleep delay.to_f
+          trigger.dispatch!(event)
+          trigger.mutex.synchronize do
+            trigger.scheduled_events.delete_if { |_, thread| thread == Thread.current }
+          end
+        rescue StandardError => e
+          trigger.logger.err(e)
+          trigger.logger.err(e.backtrace.join("\n"))
+        end
+      end
+
+      self.scheduled_events.push([event, thread])
+    end
+
+    def cancel_all_events
+      self.logger.info "Canceling all scheduled events"
+      self.mutex.synchronize do
+        self.scheduled_events.each {|_, thread| thread.kill}
+      end
+    end
+
+  end
+end

data/lib/bluepill/triggers/flapping.rb

@@ -0,0 +1,56 @@
+# -*- encoding: utf-8 -*-
+module Bluepill
+  module Triggers
+    class Flapping < Bluepill::Trigger
+      TRIGGER_STATES = [:starting, :restarting]
+
+      PARAMS = [:times, :within, :retry_in]
+
+      attr_accessor *PARAMS
+      attr_reader :timeline
+
+      def initialize(process, options = {})
+        options.reverse_merge!(:times => 5, :within => 1, :retry_in => 5)
+
+        options.each_pair do |name, val|
+          instance_variable_set("@#{name}", val) if PARAMS.include?(name)
+        end
+
+        @timeline = Util::RotationalArray.new(@times)
+        super
+      end
+
+      def notify(transition)
+        if TRIGGER_STATES.include?(transition.to_name)
+          self.timeline << Time.now.to_i
+          self.check_flapping
+        end
+      end
+
+      def reset!
+        @timeline.clear
+        super
+      end
+
+      def check_flapping
+        # The process has not flapped if we haven't encountered enough incidents
+        return unless (@timeline.compact.length == self.times)
+        
+        # Check if the incident happend within the timeframe
+        duration = (@timeline.last - @timeline.first) <= self.within
+
+        if duration
+          self.logger.info "Flapping detected: retrying in #{self.retry_in} seconds"
+
+          self.schedule_event(:start, self.retry_in) unless self.retry_in == 0 # retry_in zero means "do not retry, ever"
+          self.schedule_event(:unmonitor, 0)
+
+          @timeline.clear
+
+          # This will prevent a transition from happening in the process state_machine
+          throw :halt
+        end
+      end
+    end
+  end
+end

data/lib/bluepill/util/rotational_array.rb

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+module Bluepill
+  module Util
+    class RotationalArray < Array
+      def initialize(size)
+        @capacity = size
+
+        super() # no size - intentionally
+      end
+
+      def push(value)
+        super(value)
+
+        self.shift if self.length > @capacity
+        self
+      end
+      alias_method :<<, :push
+    end
+  end
+end