litmus_paper 0.9.6 → 0.9.7

data/README.md

@@ -20,8 +20,8 @@ Or install it yourself as:
 
 ## Usage
 
-Use the sample config to run it under unicorn. Or when running it as an
-agent-check for HAProxy use the sample xinetd config.
+Use the sample config to run it under unicorn. Or execute litmus-agent-check to
+use it as an agent-check for HAProxy.
 
 ## Contributing
 

data/bin/litmus-agent-check

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'optparse'
 require 'litmus_paper'
 require 'litmus_paper/cli/agent_check'
 

data/config.ru

@@ -1,5 +1,6 @@
 $LOAD_PATH.unshift File.expand_path('lib', File.dirname(__FILE__))
 require 'litmus_paper'
+require 'litmus_paper/app'
 
 LitmusPaper.configure
 use Rack::CommonLogger, LitmusPaper.logger

data/lib/litmus_paper/agent_check_handler.rb

@@ -0,0 +1,28 @@
+module LitmusPaper
+  class AgentCheckHandler
+    def self.handle(service)
+      output = []
+      health = LitmusPaper.check_service(service)
+      if health.nil?
+        output << "failed#NOT_FOUND"
+      else
+        case health.direction
+        when :up, :health
+          output << "ready" # administrative state
+          output << "up" # operational state
+        when :down
+          output << "drain" # administrative state
+        when :none
+          if health.ok?
+            output << "ready" # administrative state
+            output << "up" # operational state
+          else
+            output << "down" # operational state
+          end
+        end
+        output << "#{health.value.to_s}%"
+      end
+      output.join("\t")
+    end
+  end
+end

data/lib/litmus_paper/agent_check_server.rb

@@ -0,0 +1,97 @@
+# Preforking TCP server, mostly stolen from Jesse Storimer's Book Working with
+# TCP Sockets
+
+require 'socket'
+require 'litmus_paper/agent_check_handler'
+
+module LitmusPaper
+  class AgentCheckServer
+    CRLF = "\r\n"
+
+    def initialize(litmus_paper_config, services, workers, pid_file, daemonize)
+      LitmusPaper.configure(litmus_paper_config)
+      @services = services
+      @workers = workers
+      @pid_file = pid_file
+      @daemonize = daemonize
+      @control_sockets = @services.keys.map do |port|
+        TCPServer.new(port)
+      end
+      trap(:INT) { exit }
+      trap(:TERM) { exit }
+    end
+
+    # Stolen pattern from ruby Socket, modified to return the service based on
+    # the accepting port
+    def accept_loop(sockets)
+      if sockets.empty?
+        raise ArgumentError, "no sockets"
+      end
+      loop {
+        readable, _, _ = IO.select(sockets)
+        readable.each { |r|
+          begin
+            sock, addr = r.accept_nonblock
+            service = @services[r.local_address.ip_port]
+          rescue IO::WaitReadable
+            next
+          end
+          yield sock, service
+        }
+      }
+    end
+
+    def respond(sock, message)
+      sock.write(message)
+      sock.write(CRLF)
+    end
+
+    def write_pid(pid_file)
+      File.open(pid_file, 'w') do |f|
+        f.write(Process.pid)
+      end
+    end
+
+    def run
+      if @daemonize
+        Process.daemon
+      end
+      write_pid(@pid_file)
+      child_pids = []
+
+      @workers.times do
+        child_pids << spawn_child
+      end
+
+      kill_children = Proc.new { |signo|
+        child_pids.each do |cpid|
+          begin
+            Process.kill(:INT, cpid)
+          rescue Errno::ESRCH
+          end
+        end
+        File.delete(@pid_file) if File.exists?(@pid_file)
+        exit
+      }
+
+      trap(:INT, &kill_children)
+      trap(:TERM, &kill_children)
+
+      loop do
+        pid = Process.wait
+        LitmusPaper.logger.error("Process #{pid} quit unexpectedly")
+        child_pids.delete(pid)
+        child_pids << spawn_child
+      end
+    end
+
+    def spawn_child
+      fork do
+        accept_loop(@control_sockets) do |sock, service|
+          respond(sock, AgentCheckHandler.handle(service))
+          sock.close
+        end
+      end
+    end
+  end
+end

data/lib/litmus_paper/app.rb

@@ -1,3 +1,6 @@
+require 'sinatra/base'
+require 'litmus_paper/terminal_output'
+
 module LitmusPaper
   class App < Sinatra::Base
     disable :show_exceptions

data/lib/litmus_paper/cli/agent_check.rb

@@ -1,15 +1,28 @@
+require 'optparse'
+require 'litmus_paper/agent_check_server'
+
 module LitmusPaper
   module CLI
     class AgentCheck
 
       def parse_args(args)
         options = {}
+        options[:pid_file] = '/tmp/litmus-agent-check.pid'
         optparser = OptionParser.new do |opts|
-          opts.on("-s", "--service SERVICE", "Service to check") do |s|
-            options[:service] = s
+          opts.on("-s", "--service SERVICE:PORT,...", Array, "agent-check service to port mappings") do |s|
+            options[:services] = s
+          end
+          opts.on("-c", "--config CONFIG", "Path to litmus paper config file") do |c|
+            options[:litmus_paper_config] = c
+          end
+          opts.on("-p", "--pid-file PID_FILE", String, "Where to write the pid") do |p|
+            options[:pid_file] = p
           end
-          opts.on("-c", "--config CONFIG", "Path to config file") do |c|
-            options[:config] = c
+          opts.on("-w", "--workers WORKERS", Integer, "Number of worker processes") do |w|
+            options[:workers] = w
+          end
+          opts.on("-D", "--daemonize", "Daemonize the process") do |d|
+            options[:daemonize] = d
           end
           opts.on("-h", "--help", "Help text") do |h|
             options[:help] = h
@@ -29,43 +42,45 @@ module LitmusPaper
           exit 0
         end
 
-        if !options.has_key?(:service)
-          puts "Error: `-s SERVICE` required"
+        if !options.has_key?(:services)
+          puts "Error: `-s SERVICE:PORT,...` required"
           puts optparser
           exit 1
         end
-        options
-      end
 
-      def output_service_status(service, stdout)
-        output = []
-        health = LitmusPaper.check_service(service)
-        if health.nil?
-          output << "failed#NOT_FOUND"
-        else
-          case health.direction
-          when :up, :health
-            output << "ready" # administrative state
-            output << "up" # operational state
-          when :down
-            output << "drain" # administrative state
-          when :none
-            if health.ok?
-              output << "ready" # administrative state
-              output << "up" # operational state
-            else
-              output << "down" # operational state
-            end
+        if !options.has_key?(:workers) || options[:workers] <= 0
+          puts "Error: `-w WORKERS` required, and must be greater than 0"
+          puts "  Use a value equal to the number of expected concurrent"
+          puts "  agent checks from HAProxy"
+          puts optparser
+          exit 1
+        end
+
+        options[:services] = options[:services].reduce({}) do |memo, service|
+          if service.split(':').length == 2
+            service, port = service.split(':')
+            memo[port.to_i] = service
+            memo
+          else
+            puts "Error: Incorrect service port arg `-s SERVICE:PORT,...`"
+            puts optparser
+            exit 1
           end
-          output << "#{health.value.to_s}%"
         end
-        stdout.printf("%s\r\n", output.join("\t"))
+
+        options
       end
 
       def run(args)
         options = parse_args(args)
-        LitmusPaper.configure(options[:config])
-        output_service_status(options[:service], $stdout)
+        agent_check_server = LitmusPaper::AgentCheckServer.new(
+          options[:litmus_paper_config],
+          options[:services],
+          options[:workers],
+          options[:pid_file],
+          options[:daemonize],
+        )
+        agent_check_server.run
       end
 
     end

data/lib/litmus_paper/dependency/script.rb

@@ -1,3 +1,5 @@
+require 'open3'
+
 module LitmusPaper
   module Dependency
     class Script
@@ -9,23 +11,27 @@ module LitmusPaper
       end
 
       def available?
-        Timeout.timeout(@timeout) do
-          script_stdout = script_stderr = nil
-          script_status = POpen4.popen4(@command) do |stdout, stderr, stdin, pid|
-            @script_pid = pid
-            script_stdout = stdout.read.strip
-            script_stderr = stderr.read.strip
-          end
-          unless script_status.success?
-            LitmusPaper.logger.info("Available check to #{@command} failed with status #{$CHILD_STATUS.exitstatus}")
-            LitmusPaper.logger.info("Failed stdout: #{script_stdout}")
-            LitmusPaper.logger.info("Failed stderr: #{script_stderr}")
+        script_status = script_stdout = script_stderr = nil
+        Open3.popen3(@command, :pgroup=>true) do |stdin, stdout, stderr, wait_thr|
+          @script_pid = wait_thr.pid
+          thstderr = Thread.new { stderr.read }
+          thstdout = Thread.new { stdout.read }
+          if !wait_thr.join(@timeout) # wait thread does not end within timeout
+            kill_and_reap_script(-@script_pid) # kill the process group
+            raise Timeout::Error
           end
-          script_status.success?
+          script_stderr = thstderr.value
+          script_stdout = thstdout.value
+          script_status = wait_thr.value
+        end
+        unless script_status.success?
+          LitmusPaper.logger.info("Available check to #{@command} failed with status #{script_status.exitstatus}")
+          LitmusPaper.logger.info("Failed stdout: #{script_stdout}")
+          LitmusPaper.logger.info("Failed stderr: #{script_stderr}")
         end
+        script_status.success?
       rescue Timeout::Error
         LitmusPaper.logger.info("Timeout running command: '#{@command}'")
-        kill_and_reap_script(@script_pid)
         false
       rescue => e
         LitmusPaper.logger.info("Available check to #{@uri} failed with #{e.message}")

data/lib/litmus_paper/metric/cpu_load.rb

@@ -1,9 +1,8 @@
 module LitmusPaper
   module Metric
     class CPULoad
-      def initialize(weight, facter = Facter)
+      def initialize(weight)
         @weight = weight
-        @facter = facter
       end
 
       def current_health
@@ -11,11 +10,18 @@ module LitmusPaper
       end
 
       def processor_count
-        @processor_count ||= @facter.value('processorcount').to_i
+        @processor_count ||= File.readlines('/proc/cpuinfo').reduce(0) do |memo, line|
+          if line =~ /^processor/
+            memo + 1
+          else
+            memo
+          end
+        end
+
       end
 
       def load_average
-        @facter.value('loadaverage').split(' ').first.to_f
+        File.read('/proc/loadavg').split(' ').first.to_f
       end
 
       def to_s

data/lib/litmus_paper/metric/script.rb

@@ -1,3 +1,5 @@
+require 'open3'
+
 module LitmusPaper
   module Metric
     class Script
@@ -15,24 +17,28 @@ module LitmusPaper
 
       def result
         value = 0
-        Timeout.timeout(@timeout) do
-          script_stdout = script_stderr = nil
-          script_status = POpen4.popen4(@command) do |stdout, stderr, stdin, pid|
-            @script_pid = pid
-            value = script_stdout = stdout.read.strip
-            script_stderr = stderr.read.strip
-          end
-          unless script_status.success?
-            LitmusPaper.logger.info("Available check to #{@command} failed with status #{$CHILD_STATUS.exitstatus}")
-            LitmusPaper.logger.info("Failed stdout: #{script_stdout}")
-            LitmusPaper.logger.info("Failed stderr: #{script_stderr}")
+        script_status = script_stdout = script_stderr = nil
+        Open3.popen3(@command, :pgroup=>true) do |stdin, stdout, stderr, wait_thr|
+          @script_pid = wait_thr.pid
+          thstderr = Thread.new { stderr.read }
+          thstdout = Thread.new { stdout.read }
+          if !wait_thr.join(@timeout) # wait thread does not end within timeout
+            kill_and_reap_script(-@script_pid) # kill the process group
+            raise Timeout::Error
           end
-
-          value.to_f
+          script_stderr = thstderr.value
+          script_stdout = thstdout.value
+          value = script_stdout.strip
+          script_status = wait_thr.value
+        end
+        unless script_status.success?
+          LitmusPaper.logger.info("Available check to #{@command} failed with status #{script_status.exitstatus}")
+          LitmusPaper.logger.info("Failed stdout: #{script_stdout}")
+          LitmusPaper.logger.info("Failed stderr: #{script_stderr}")
         end
+        value.to_f
       rescue Timeout::Error
         LitmusPaper.logger.info("Available check to '#{@command}' timed out")
-        kill_and_reap_script(@script_pid)
         0
       end
 

data/lib/litmus_paper/terminal_output.rb

@@ -1,4 +1,6 @@
 # encoding: UTF-8
+require 'colorize'
+
 module LitmusPaper
   class TerminalOutput
     def self.service_status

data/lib/litmus_paper/version.rb

@@ -1,3 +1,3 @@
 module LitmusPaper
-  VERSION = "0.9.6"
+  VERSION = "0.9.7"
 end

data/lib/litmus_paper.rb

@@ -1,11 +1,3 @@
-require 'pathname'
-require 'net/http'
-require 'net/https'
-require 'uri'
-require 'forwardable'
-require 'English'
-require 'colorize'
-
 # Ruby's stock DNS resolution, by default, blocks the entire Ruby VM from
 # processing while the lookup is happening, because it calls out to the native
 # libc resolver code. A slow DNS server can cause your entire Ruby process to
@@ -15,15 +7,13 @@ require 'colorize'
 # 'resolv-replace' monkeypatches the various Ruby Socket objects to use resolv
 #
 require 'resolv-replace'
+require 'net/http'
+require 'net/https'
+require 'uri'
+require 'forwardable'
 
-require 'popen4'
-require 'sinatra/base'
-require 'facter'
 require 'remote_syslog_logger'
 
-require 'facts/loadaverage'
-
-require 'litmus_paper/app'
 require 'litmus_paper/configuration'
 require 'litmus_paper/configuration_file'
 require 'litmus_paper/dependency/file_contents'
@@ -33,7 +23,6 @@ require 'litmus_paper/dependency/script'
 require 'litmus_paper/dependency/tcp'
 require 'litmus_paper/health'
 require 'litmus_paper/logger'
-require 'litmus_paper/metric/available_memory'
 require 'litmus_paper/metric/big_brother_service'
 require 'litmus_paper/metric/constant_metric'
 require 'litmus_paper/metric/cpu_load'
@@ -41,7 +30,6 @@ require 'litmus_paper/metric/internet_health'
 require 'litmus_paper/metric/script'
 require 'litmus_paper/service'
 require 'litmus_paper/status_file'
-require 'litmus_paper/terminal_output'
 require 'litmus_paper/version'
 
 module LitmusPaper
@@ -54,8 +42,6 @@ module LitmusPaper
   self.logger = Logger.new
 
   def self.check_service(service_name)
-    Facter.flush
-
     if service = services[service_name]
       service.current_health
     else

data/litmus-agent-check.init.sh

@@ -0,0 +1,130 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          litmus-agent-check
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Litmus agent-check for HAProxy
+# Description:       Provide an agent-check server for HAProxy
+### END INIT INFO
+
+# Author: Braintreeps <code@getbraintree.com>
+
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="Litmus HAProxy agent-check server"
+NAME=litmus-agent-check
+DAEMON=/usr/bin/$NAME
+PIDFILE=/var/run/litmus-agent-check/pid
+DAEMON_ARGS="-D -p ${PIDFILE}"
+SCRIPTNAME=/etc/init.d/$NAME
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/default/$NAME ] && . /etc/default/$NAME
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
+. /lib/lsb/init-functions
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+  mkdir -p $(dirname $PIDFILE)
+	# Return
+	#   0 if daemon has been started
+	#   1 if daemon was already running
+	#   2 if daemon could not be started
+	start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test > /dev/null \
+		|| return 1
+	start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON -- \
+		$DAEMON_ARGS $EXTRA_ARGS\
+		|| return 2
+	# Add code here, if necessary, that waits for the process to be ready
+	# to handle requests from services started subsequently which depend
+	# on this one.  As a last resort, sleep for some time.
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+	# Return
+	#   0 if daemon has been stopped
+	#   1 if daemon was already stopped
+	#   2 if daemon could not be stopped
+	#   other if a failure occurred
+	start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE
+	RETVAL="$?"
+	[ "$RETVAL" = 2 ] && return 2
+	# Many daemons don't delete their pidfiles when they exit.
+	rm -f $PIDFILE
+	return "$RETVAL"
+}
+
+#
+# Function that sends a SIGHUP to the daemon/service
+#
+do_reload() {
+	#
+	# If the daemon can reload its configuration without
+	# restarting (for example, when it is sent a SIGHUP),
+	# then implement that here.
+	#
+	start-stop-daemon --stop --signal HUP --quiet --pidfile $PIDFILE
+	return 0
+}
+
+case "$1" in
+  start)
+    [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
+    do_start
+    case "$?" in
+      0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
+      2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
+    esac
+	;;
+  stop)
+    [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
+    do_stop
+    case "$?" in
+      0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
+      2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
+    esac
+	;;
+  status)
+    status_of_proc -p ${PIDFILE} && exit 0 || exit $?
+  ;;
+  reload|force-reload)
+    log_daemon_msg "Reloading $DESC" "$NAME"
+    do_reload
+    log_end_msg $?
+	;;
+  restart)
+    log_daemon_msg "Restarting $DESC" "$NAME"
+    do_stop
+    case "$?" in
+      0|1)
+      do_start
+      case "$?" in
+        0) log_end_msg 0 ;;
+        1) log_end_msg 1 ;; # Old process is still running
+        *) log_end_msg 1 ;; # Failed to start
+      esac
+      ;;
+      *)
+        # Failed to stop
+      log_end_msg 1
+      ;;
+    esac
+	;;
+  *)
+    echo "Usage: $SCRIPTNAME {start|stop|status|restart|reload|force-reload}" >&2
+    exit 3
+	;;
+esac

data/litmus_paper.gemspec

@@ -17,9 +17,7 @@ Gem::Specification.new do |gem|
   gem.version       = LitmusPaper::VERSION
 
   gem.add_dependency "sinatra", "~> 1.3.2"
-  gem.add_dependency "facter", "~> 1.7.5"
   gem.add_dependency "remote_syslog_logger", "~> 1.0.3"
-  gem.add_dependency "popen4", "~> 0.1.2"
   gem.add_dependency "unicorn", "~> 4.6.2"
   gem.add_dependency "colorize"
 

data/spec/litmus_paper/{cli/agent_check_spec.rb → agent_check_handler_spec.rb}

@@ -1,47 +1,40 @@
 require 'spec_helper'
-require 'litmus_paper/cli/agent_check'
+require 'litmus_paper/agent_check_handler'
 
-describe LitmusPaper::CLI::AgentCheck do
+describe LitmusPaper::AgentCheckHandler do
   before :each do
     LitmusPaper.configure(TEST_CONFIG)
   end
 
-  def agent_check(service)
-    output = StringIO.new
-    LitmusPaper::CLI::AgentCheck.new.output_service_status(service, output)
-    output.rewind
-    output.readline
-  end
-
   describe "output_service_status" do
     it "returns the forced health value for a healthy service" do
       test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [LitmusPaper::Metric::ConstantMetric.new(100)])
       LitmusPaper.services['test'] = test_service
       LitmusPaper::StatusFile.service_health_file("test").create("Forcing health", 88)
-      agent_check("test").should == "ready\tup\t88%\r\n"
+      LitmusPaper::AgentCheckHandler.handle("test").should == "ready\tup\t88%"
     end
 
     it "returns the actual health value for an unhealthy service when the measured health is less than the forced value" do
       test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [LitmusPaper::Metric::ConstantMetric.new(100)])
       LitmusPaper.services['test'] = test_service
       LitmusPaper::StatusFile.service_health_file("test").create("Forcing health", 88)
-      agent_check("test").should == "ready\tup\t0%\r\n"
+      LitmusPaper::AgentCheckHandler.handle("test").should == "ready\tup\t0%"
     end
 
     it "is 'ready' when the service is passing" do
       test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [LitmusPaper::Metric::ConstantMetric.new(100)])
       LitmusPaper.services['test'] = test_service
-      agent_check("test").should == "ready\tup\t100%\r\n"
+      LitmusPaper::AgentCheckHandler.handle("test").should == "ready\tup\t100%"
     end
 
     it "is 'down' when the check fails" do
       test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [LitmusPaper::Metric::ConstantMetric.new(100)])
       LitmusPaper.services['test'] = test_service
-      agent_check("test").should == "down\t0%\r\n"
+      LitmusPaper::AgentCheckHandler.handle("test").should == "down\t0%"
     end
 
     it "is 'failed' when the service is unknown" do
-      agent_check("unknown").should == "failed#NOT_FOUND\r\n"
+      LitmusPaper::AgentCheckHandler.handle("unknown").should == "failed#NOT_FOUND"
     end
 
     it "is 'drain' when an up file and down file exists" do
@@ -49,7 +42,7 @@ describe LitmusPaper::CLI::AgentCheck do
       LitmusPaper.services['test'] = test_service
       LitmusPaper::StatusFile.service_up_file("test").create("Up for testing")
       LitmusPaper::StatusFile.service_down_file("test").create("Down for testing")
-      agent_check("test").should == "drain\t0%\r\n"
+      LitmusPaper::AgentCheckHandler.handle("test").should == "drain\t0%"
     end
 
     it "is 'drain' when a global down file and up file exists" do
@@ -57,28 +50,28 @@ describe LitmusPaper::CLI::AgentCheck do
       LitmusPaper.services['test'] = test_service
       LitmusPaper::StatusFile.global_down_file.create("Down for testing")
       LitmusPaper::StatusFile.global_up_file.create("Up for testing")
-      agent_check("test").should == "drain\t0%\r\n"
+      LitmusPaper::AgentCheckHandler.handle("test").should == "drain\t0%"
     end
 
     it "is 'ready' when an up file exists" do
       test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [LitmusPaper::Metric::ConstantMetric.new(100)])
       LitmusPaper.services['test'] = test_service
       LitmusPaper::StatusFile.service_up_file("test").create("Up for testing")
-      agent_check("test").should == "ready\tup\t100%\r\n"
+      LitmusPaper::AgentCheckHandler.handle("test").should == "ready\tup\t100%"
     end
 
     it "is 'drain' when a global down file exists" do
       test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [LitmusPaper::Metric::ConstantMetric.new(100)])
       LitmusPaper.services['test'] = test_service
       LitmusPaper::StatusFile.global_down_file.create("Down for testing")
-      agent_check("test").should == "drain\t0%\r\n"
+      LitmusPaper::AgentCheckHandler.handle("test").should == "drain\t0%"
     end
 
     it "is 'ready' when a global up file exists" do
       test_service = LitmusPaper::Service.new('test', [NeverAvailableDependency.new], [LitmusPaper::Metric::ConstantMetric.new(100)])
       LitmusPaper.services['test'] = test_service
       LitmusPaper::StatusFile.global_up_file.create("Up for testing")
-      agent_check("test").should == "ready\tup\t100%\r\n"
+      LitmusPaper::AgentCheckHandler.handle("test").should == "ready\tup\t100%"
     end
   end
 end

data/spec/litmus_paper/agent_check_server_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+require 'litmus_paper/agent_check_server'
+
+describe LitmusPaper::AgentCheckServer do
+  pid = nil
+
+  before :all do
+    if ! Kernel.system("bundle exec litmus-agent-check -s passing_test:9191,test:9192 -c spec/support/test.config -w 10 -D")
+      fail('Unable to start server')
+    end
+    port_open = false
+    while ! port_open do
+      begin
+        TCPSocket.new('localhost', 9191)
+      rescue StandardError => e
+        sleep 0.1
+        next
+      end
+      port_open = true
+    end
+  end
+
+  after :all do
+    Process.kill(:TERM, File.read('/tmp/litmus-agent-check.pid').to_i)
+  end
+
+  describe "The agent-check text protocol" do
+    it "returns the health from a passing test" do
+      TCPSocket.open('localhost', 9191) do |s|
+        s.gets.should match(/ready\tup\t\d+%\r\n/)
+      end
+    end
+    it "returns the health from a failing test" do
+      TCPSocket.open('localhost', 9192) do |s|
+        s.gets.should match(/down\t0%\r\n/)
+      end
+    end
+  end
+
+  describe "server" do
+    it "has the configured number of children running" do
+      pid = File.read('/tmp/litmus-agent-check.pid').to_i
+      children = `ps --no-headers --ppid #{pid}|wc -l`
+      children.strip.to_i == 10
+    end
+  end
+
+  describe "server" do
+    it "if a child dies you get a new one" do
+      pid = File.read('/tmp/litmus-agent-check.pid').to_i
+      Kernel.system("kill -9 $(ps --no-headers --ppid #{pid} -o pid=|tail -1)")
+      sleep 0.5
+      children = `ps --no-headers --ppid #{pid}|wc -l`
+      children.strip.to_i == 10
+    end
+  end
+end

data/spec/litmus_paper/app_spec.rb

@@ -418,22 +418,6 @@ describe LitmusPaper::App do
       last_response.headers["X-Health-Forced"].should == "up"
       last_response.body.should match(/Up for testing/)
     end
-
-    it "resets the Facter cache" do
-      test_service = LitmusPaper::Service.new('test', [AlwaysAvailableDependency.new], [LitmusPaper::Metric::ConstantMetric.new(100)])
-      LitmusPaper.services['test'] = test_service
-
-      get "/test/status"
-      last_response.should be_ok
-
-      facter_uptime = Facter.value("uptime_seconds")
-      sleep 1
-
-      get "/test/status"
-      last_response.should be_ok
-
-      Facter.value("uptime_seconds").should > facter_uptime
-    end
   end
 
   describe "server errors" do

data/spec/litmus_paper/cli/admin_spec.rb

@@ -33,7 +33,7 @@ describe 'litmusctl' do
   describe 'status' do
     it 'returns the status of a service' do
       _litmusctl('status test').should match("Health: 0")
-      _litmusctl('status passing_test').should match(/Health: \d\d/)
+      _litmusctl('status passing_test').should match(/Health: \d+/)
     end
 
     it "returns 'NOT FOUND' for a service that doesn't exist" do
@@ -79,7 +79,7 @@ describe 'litmusctl' do
       _litmusctl('force up test -d').should match("File deleted")
 
       status = _litmusctl('status passing_test')
-      status.should match(/Health: \d\d/)
+      status.should match(/Health: \d+/)
       status.should_not match(/for testing/)
     end
 
@@ -87,7 +87,7 @@ describe 'litmusctl' do
       _litmusctl('force health 88 test -r "for testing"').should match("File created")
       _litmusctl('force health test -d').should match("File deleted")
       status = _litmusctl('status passing_test')
-      status.should match(/Health: \d\d/)
+      status.should match(/Health: \d+/)
       status.should_not match(/for testing/)
     end
 

data/spec/litmus_paper/dependency/script_spec.rb

@@ -7,8 +7,28 @@ describe LitmusPaper::Dependency::Script do
       check.should be_available
     end
 
+    it "is true when the script returns a lot of data" do
+      check = LitmusPaper::Dependency::Script.new("dd if=/dev/urandom bs=1M count=1|base64")
+      check.should be_available
+    end
+
     it "is false when the script returns 1" do
-      check = LitmusPaper::Dependency::Script.new("false")
+      check = LitmusPaper::Dependency::Script.new("echo Hello && echo Goodbye 1>&2 && false")
+      check.should_not be_available
+    end
+
+    it "logs stdout and stderr when it fails" do
+      check = LitmusPaper::Dependency::Script.new("echo Hello && echo Goodbye 1>&2 && false")
+      count = 0
+      logs = [
+        "Available check to echo Hello && echo Goodbye 1>&2 && false failed with status 1",
+        "Failed stdout: Hello\n",
+        "Failed stderr: Goodbye\n",
+      ]
+      LitmusPaper.logger.should_receive(:info).exactly(3).times do |log|
+        log.should == logs[count]
+        count += 1
+      end
       check.should_not be_available
     end
 

data/spec/litmus_paper/metric/cpu_load_spec.rb

@@ -3,14 +3,20 @@ require 'spec_helper'
 describe LitmusPaper::Metric::CPULoad do
   describe "#current_health" do
     it "is the percent of available cpu capacity" do
-      facter = StubFacter.new({"processorcount" => "4", "loadaverage" => "1.00 0.40 0.10"})
-      cpu_load = LitmusPaper::Metric::CPULoad.new(40, facter)
+      LitmusPaper::Metric::CPULoad.any_instance.stub(
+        :processor_count => 4,
+        :load_average => 1.00,
+      )
+      cpu_load = LitmusPaper::Metric::CPULoad.new(40)
       cpu_load.current_health.should == 30
     end
 
     it "is one when the load is above one per core" do
-      facter = StubFacter.new({"processorcount" => "4", "loadaverage" => "20.00 11.40 10.00"})
-      cpu_load = LitmusPaper::Metric::CPULoad.new(50, facter)
+      LitmusPaper::Metric::CPULoad.any_instance.stub(
+        :processor_count => 4,
+        :load_average => 20.00,
+      )
+      cpu_load = LitmusPaper::Metric::CPULoad.new(50)
       cpu_load.current_health.should == 1
     end
   end
@@ -22,7 +28,7 @@ describe LitmusPaper::Metric::CPULoad do
     end
 
     it "is cached" do
-      Facter.should_receive(:value).once.and_return("10")
+      File.should_receive(:readlines).with('/proc/cpuinfo').once.and_return(["processor       : 0\n"])
       cpu_load = LitmusPaper::Metric::CPULoad.new(50)
       cpu_load.processor_count
       cpu_load.processor_count
@@ -35,6 +41,13 @@ describe LitmusPaper::Metric::CPULoad do
       cpu_load = LitmusPaper::Metric::CPULoad.new(50)
       cpu_load.load_average.should > 0.0
     end
+
+    it "is not cached" do
+      File.should_receive(:read).with('/proc/loadavg').twice.and_return("0.08 0.12 0.15 2/1190 9152\n")
+      cpu_load = LitmusPaper::Metric::CPULoad.new(50)
+      cpu_load.load_average.should > 0.0
+      cpu_load.load_average.should > 0.0
+    end
   end
 
   describe "#to_s" do

data/spec/litmus_paper/metric/script_spec.rb

@@ -7,6 +7,26 @@ describe LitmusPaper::Metric::Script do
       check.current_health.should == 1
     end
 
+    it "is true when the script returns a lot of data" do
+      check = LitmusPaper::Metric::Script.new("dd if=/dev/urandom bs=1M count=1|base64 >&2 && echo 1", 1)
+      check.current_health.should == 1
+    end
+
+    it "logs stdout and stderr when it fails" do
+      check = LitmusPaper::Metric::Script.new("echo Hello && echo Goodbye 1>&2 && false", 1)
+      count = 0
+      logs = [
+        "Available check to echo Hello && echo Goodbye 1>&2 && false failed with status 1",
+        "Failed stdout: Hello\n",
+        "Failed stderr: Goodbye\n",
+      ]
+      LitmusPaper.logger.should_receive(:info).exactly(3).times do |log|
+        log.should == logs[count]
+        count += 1
+      end
+      check.current_health.should == 0
+    end
+
     it "is zero when the script exits 1" do
       check = LitmusPaper::Metric::Script.new("false", 1)
       check.current_health.should == 0

data/spec/spec_helper.rb

@@ -3,6 +3,7 @@ ENV['RACK_ENV'] = 'test'
 require 'rspec'
 require 'rack/test'
 require 'litmus_paper'
+require 'litmus_paper/app'
 require 'tempfile'
 
 TEST_CONFIG_DIR = "/tmp/litmus_paper"

data/spec/support/config.d/passing_test.config

@@ -2,5 +2,4 @@
 
 service :passing_test do |s|
   s.measure_health Metric::CPULoad, :weight => 50
-  s.measure_health Metric::AvailableMemory, :weight => 50
 end

data/spec/support/config.d/test.config

@@ -4,5 +4,4 @@ service :test do |s|
   s.depends Dependency::HTTP, "http://localhost/heartbeat"
 
   s.measure_health Metric::CPULoad, :weight => 50
-  s.measure_health Metric::AvailableMemory, :weight => 50
 end

data/spec/support/test.config

@@ -8,10 +8,9 @@ service :test do |s|
   s.depends Dependency::HTTP, "http://localhost/heartbeat"
 
   s.measure_health Metric::CPULoad, :weight => 50
-  s.measure_health Metric::AvailableMemory, :weight => 50
 end
 
 service :passing_test do |s|
   s.measure_health Metric::CPULoad, :weight => 50
-  s.measure_health Metric::AvailableMemory, :weight => 50
+  s.depends Dependency::Script, "/bin/true"
 end

data/spec/support/test.reload.config

@@ -8,10 +8,8 @@ service :foo do |s|
   s.depends Dependency::HTTP, "http://localhost/heartbeat"
 
   s.measure_health Metric::CPULoad, :weight => 50
-  s.measure_health Metric::AvailableMemory, :weight => 50
 end
 
 service :passing_test do |s|
   s.measure_health Metric::CPULoad, :weight => 50
-  s.measure_health Metric::AvailableMemory, :weight => 50
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: litmus_paper
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 0.9.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-17 00:00:00.000000000 Z
+date: 2017-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
@@ -27,22 +27,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.3.2
-- !ruby/object:Gem::Dependency
-  name: facter
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.7.5
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.7.5
 - !ruby/object:Gem::Dependency
   name: remote_syslog_logger
   requirement: !ruby/object:Gem::Requirement
@@ -59,22 +43,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.0.3
-- !ruby/object:Gem::Dependency
-  name: popen4
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.1.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: unicorn
   requirement: !ruby/object:Gem::Requirement
@@ -193,8 +161,9 @@ files:
 - bin/litmus-agent-check
 - bin/litmusctl
 - config.ru
-- lib/facts/loadaverage.rb
 - lib/litmus_paper.rb
+- lib/litmus_paper/agent_check_handler.rb
+- lib/litmus_paper/agent_check_server.rb
 - lib/litmus_paper/app.rb
 - lib/litmus_paper/cli/admin.rb
 - lib/litmus_paper/cli/admin/command.rb
@@ -212,7 +181,6 @@ files:
 - lib/litmus_paper/dependency/tcp.rb
 - lib/litmus_paper/health.rb
 - lib/litmus_paper/logger.rb
-- lib/litmus_paper/metric/available_memory.rb
 - lib/litmus_paper/metric/big_brother_service.rb
 - lib/litmus_paper/metric/constant_metric.rb
 - lib/litmus_paper/metric/cpu_load.rb
@@ -222,10 +190,12 @@ files:
 - lib/litmus_paper/status_file.rb
 - lib/litmus_paper/terminal_output.rb
 - lib/litmus_paper/version.rb
+- litmus-agent-check.init.sh
 - litmus_paper.gemspec
+- spec/litmus_paper/agent_check_handler_spec.rb
+- spec/litmus_paper/agent_check_server_spec.rb
 - spec/litmus_paper/app_spec.rb
 - spec/litmus_paper/cli/admin_spec.rb
-- spec/litmus_paper/cli/agent_check_spec.rb
 - spec/litmus_paper/configuration_file_spec.rb
 - spec/litmus_paper/dependency/file_contents_spec.rb
 - spec/litmus_paper/dependency/haproxy_backends_spec.rb
@@ -233,7 +203,6 @@ files:
 - spec/litmus_paper/dependency/script_spec.rb
 - spec/litmus_paper/dependency/tcp_spec.rb
 - spec/litmus_paper/health_spec.rb
-- spec/litmus_paper/metric/available_memory_spec.rb
 - spec/litmus_paper/metric/big_brother_service_spec.rb
 - spec/litmus_paper/metric/constant_metric_spec.rb
 - spec/litmus_paper/metric/cpu_load_spec.rb
@@ -254,12 +223,10 @@ files:
 - spec/support/http_test_server_config.ru
 - spec/support/never_available_dependency.rb
 - spec/support/stdout_logger.rb
-- spec/support/stub_facter.rb
 - spec/support/test.config
 - spec/support/test.d.config
 - spec/support/test.reload.config
 - spec/support/test.unicorn.config
-- xinetd.conf
 homepage: https://github.com/braintree/litmus_paper
 licenses: []
 post_install_message: 
@@ -285,9 +252,10 @@ signing_key:
 specification_version: 3
 summary: Backend health tester for HA Services, partner project of big_brother
 test_files:
+- spec/litmus_paper/agent_check_handler_spec.rb
+- spec/litmus_paper/agent_check_server_spec.rb
 - spec/litmus_paper/app_spec.rb
 - spec/litmus_paper/cli/admin_spec.rb
-- spec/litmus_paper/cli/agent_check_spec.rb
 - spec/litmus_paper/configuration_file_spec.rb
 - spec/litmus_paper/dependency/file_contents_spec.rb
 - spec/litmus_paper/dependency/haproxy_backends_spec.rb
@@ -295,7 +263,6 @@ test_files:
 - spec/litmus_paper/dependency/script_spec.rb
 - spec/litmus_paper/dependency/tcp_spec.rb
 - spec/litmus_paper/health_spec.rb
-- spec/litmus_paper/metric/available_memory_spec.rb
 - spec/litmus_paper/metric/big_brother_service_spec.rb
 - spec/litmus_paper/metric/constant_metric_spec.rb
 - spec/litmus_paper/metric/cpu_load_spec.rb
@@ -316,7 +283,6 @@ test_files:
 - spec/support/http_test_server_config.ru
 - spec/support/never_available_dependency.rb
 - spec/support/stdout_logger.rb
-- spec/support/stub_facter.rb
 - spec/support/test.config
 - spec/support/test.d.config
 - spec/support/test.reload.config

data/lib/facts/loadaverage.rb

@@ -1,6 +0,0 @@
-Facter.add("loadaverage") do
-  setcode do
-    uptime = Facter::Util::Resolution.exec("uptime")
-    uptime.split(":").last
-  end
-end

data/lib/litmus_paper/metric/available_memory.rb

@@ -1,36 +0,0 @@
-module LitmusPaper
-  module Metric
-    class AvailableMemory
-      MULTIPLIER = {
-        "GB" => 1024*1024*1024,
-        "MB" => 1024*1024,
-        "KB" => 1024
-      }
-
-      def initialize(weight, facter = Facter)
-        @weight = weight
-        @facter = facter
-      end
-
-      def current_health
-        @weight * memory_free / memory_total
-      end
-
-      def memory_total
-        return @memory_total unless @memory_total.nil?
-
-        size, scale = @facter.value('memorysize').split(' ')
-        @memory_total = (size.to_f * MULTIPLIER[scale]).to_i
-      end
-
-      def memory_free
-        size, scale = @facter.value('memoryfree').split(' ')
-        (size.to_f * MULTIPLIER[scale]).to_i
-      end
-
-      def to_s
-        "Metric::AvailableMemory(#{@weight})"
-      end
-    end
-  end
-end

data/spec/litmus_paper/metric/available_memory_spec.rb

@@ -1,58 +0,0 @@
-require 'spec_helper'
-
-describe LitmusPaper::Metric::AvailableMemory do
-  describe "#current_health" do
-    it "multiplies weight by memory available" do
-      facter = StubFacter.new({"memorysize" => "10 GB", "memoryfree" => "5 GB"})
-      memory = LitmusPaper::Metric::AvailableMemory.new(50, facter)
-      memory.current_health.should == 25
-    end
-
-    it "multiplies weight by memory available when handling floating point values" do
-      facter = StubFacter.new({"memorysize" => "2.0 GB", "memoryfree" => "1.8 GB"})
-      memory = LitmusPaper::Metric::AvailableMemory.new(50, facter)
-      memory.current_health.should == 44
-    end
-
-    describe "#memory_total" do
-      it "is a positive integer" do
-        metric = LitmusPaper::Metric::AvailableMemory.new(50)
-        metric.memory_total.should > 1_000
-      end
-
-      it "handles floating point values properly" do
-        facter = StubFacter.new("memorysize" => "1.80 GB")
-        memory = LitmusPaper::Metric::AvailableMemory.new(50, facter)
-        memory.memory_total.should == 1932735283
-      end
-
-      it "is cached" do
-        Facter.should_receive(:value).once.and_return("10 MB")
-        metric = LitmusPaper::Metric::AvailableMemory.new(50)
-        metric.memory_total
-        metric.memory_total
-        metric.memory_total
-      end
-    end
-
-    describe "#memory_free" do
-      it "is a positive integer" do
-        metric = LitmusPaper::Metric::AvailableMemory.new(50)
-        metric.memory_free.should > 100
-      end
-
-      it "handles floating point values properly" do
-        facter = StubFacter.new("memoryfree" => "1.80 GB")
-        memory = LitmusPaper::Metric::AvailableMemory.new(50, facter)
-        memory.memory_free.should == 1932735283
-      end
-    end
-
-    describe "#to_s" do
-      it "is the name of the check and the max weight" do
-        metric = LitmusPaper::Metric::AvailableMemory.new(50)
-        metric.to_s.should == "Metric::AvailableMemory(50)"
-      end
-    end
-  end
-end

data/spec/support/stub_facter.rb

@@ -1,9 +0,0 @@
-class StubFacter
-  def initialize(values)
-    @values = values
-  end
-
-  def value(key)
-    @values[key]
-  end
-end

data/xinetd.conf

@@ -1,13 +0,0 @@
-service internet_health
-{
-  disable     = no
-  socket_type = stream
-  protocol    = tcp
-  port        = 9191
-  user        = agent-check
-  wait        = no
-  server      = /usr/bin/litmus-agent-check
-  server_args = -s isp
-  type        = UNLISTED
-  instances   = 10
-}