instrumental_tools 1.0.0.rc2 → 1.0.0

data/chef/instrumental_tools/recipes/default.rb

@@ -0,0 +1,21 @@
+packagecloud_repo "expectedbehavior/instrumental" do
+  case node["platform_family"]
+  when "debian"
+    type "deb"
+  when "rhel"
+    type "rpm"
+  end
+end
+
+package "instrumental-tools" do
+  action :upgrade
+end
+
+template "/etc/instrumental.yml" do
+  source "instrumental.yml.erb"
+  mode   "0440"
+  owner  "nobody"
+  variables(
+    :api_key => node[:instrumental][:api_key]
+  )
+end

data/chef/instrumental_tools/templates/instrumental.yml.erb

@@ -0,0 +1,6 @@
+---
+# Replace YOUR_API_KEY with the API key for your Instrumental project and
+# remove the # from the beginning of the following line to have instrument_server
+# start sending metrics to your account.
+
+api_key: <%= @api_key %>

data/conf/instrumental.yml

@@ -0,0 +1,6 @@
+---
+# Replace YOUR_API_KEY with the API key for your Instrumental project and
+# remove the # from the beginning of the following line to have instrument_server
+# start sending metrics to your account.
+
+#api_key: YOUR_API_KEY

data/debian/after-install.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+update-rc.d instrument_server defaults
+/etc/init.d/instrument_server start
+echo "Remember to edit /etc/instrumental.yml with your Instrumental API key"
+exit 0

data/debian/after-remove.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+update-rc.d -f instrument_server remove
+exit 0

data/debian/before-remove.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+/etc/init.d/instrument_server stop
+exit 0

data/debian/instrument_server

@@ -0,0 +1,46 @@
+#! /bin/sh
+### BEGIN INIT INFO
+# Provides:          instrument_server
+# Required-Start:    $all
+# Required-Stop:     $all
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start instrument_server at boot to provide system metrics
+# Description:       Report system level metrics to the Instrumental service (https://instrumentalapp.com/)
+### END INIT INFO
+
+set -e
+
+DIRECTORY="/opt/instrumental-tools/"
+CONFIG_FILE="/etc/instrumental.yml"
+TMPDIR=$DIRECTORY
+PID="${DIRECTORY}instrument_server.pid"
+LOG="${DIRECTORY}instrument_server.log"
+SCRIPT_LOCATION="${DIRECTORY}.instrumental_scripts"
+USER_TO_RUN_AS="nobody"
+ARGS="-f ${CONFIG_FILE} -p ${PID} -l ${LOG} -s ${SCRIPT_LOCATION} -u ${USER_TO_RUN_AS} -t ${TMPDIR}"
+PROCESS="${DIRECTORY}instrument_server ${ARGS}"
+
+case "$1" in
+  start)
+    $PROCESS start
+    ;;
+  stop)
+    $PROCESS stop
+    ;;
+  restart)
+    $PROCESS restart
+    ;;
+  status)
+    $PROCESS status
+    ;;
+  force-reload)
+    $PROCESS stop && $PROCESS clean && $PROCESS start
+    ;;
+  *)
+    echo "Usage: /etc/init.d/instrumental-tools {start|stop|restart|status}"
+    exit 1
+    ;;
+esac
+
+exit 0

data/examples/mysql/mysql_status.rb

@@ -9,8 +9,8 @@ MYSQL_USER              = ENV["MYSQL_USER"]
 MYSQL_DEFAULTS_FILE     = ENV["MYSQL_DEFAULTS_FILE"]
 MYSQL_PASSWORD          = ENV["MYSQL_PASSWORD"]
 
-RATE_METRICS_TO_INSPECT = %w{Queries Bytes_sent Bytes_received Connections Slow_queries}
-CANARY_METRIC           = "Queries"
+RATE_METRICS_TO_INSPECT = %w{queries bytes_sent bytes_received connections slow_queries}
+CANARY_METRIC           = "queries"
 
 env  = {}
 args = []
@@ -61,7 +61,7 @@ if !exit_status.success?
 else
   output = stdout_r.read.lines                                         # each line
                         .map { |line| line.chomp.split }              # split by space characters
-                        .map { |(name, value, _)| [name, value.to_f] } # with values coerced to floats
+                        .map { |(name, value, _)| [name.downcase, value.to_f] } # with values coerced to floats
   stats  = Hash[output]
   if (stats[CANARY_METRIC] < previous_values[CANARY_METRIC].to_i) || previous_values[CANARY_METRIC].nil?
     # The server has restarted, don't trust previous values for calculating difference

data/instrumental_tools.gemspec

@@ -1,18 +1,37 @@
 $: << "./lib"
+require 'find'
 require 'instrumental_tools/version'
 
+gitignore = Array(File.exists?(".gitignore") ? File.read(".gitignore").split("\n") : []) + [".git", ".gitignore"]
+all_files = []
+
+Find.find(".") do |path|
+  scrubbed_path = path.gsub(/\A\.\//, "")
+  if gitignore.any? { |glob| File.fnmatch(glob, scrubbed_path) }
+    Find.prune
+  else
+    if !File.directory?(scrubbed_path)
+      all_files << scrubbed_path
+    end
+  end
+end
+
+test_files = all_files.select { |path| path =~ /\A(test|spec|features)\//i }
+bin_files  = all_files.select { |path| path.index("bin") == 0 }.map { |path| File.basename(path) }
+
 Gem::Specification.new do |s|
   s.name        = "instrumental_tools"
   s.version     = Instrumental::Tools::VERSION
-  s.authors     = ["Elijah Miller", "Christopher Zelenak", "Kristopher Chambers", "Matthew Hassfurder"]
+  s.authors     = ["Expected Behavior"]
   s.email       = ["support@instrumentalapp.com"]
   s.homepage    = "http://github.com/expectedbehavior/instrumental_tools"
   s.summary     = %q{Command line tools for Instrumental}
   s.description = %q{A collection of scripts useful for monitoring servers and services with Instrumental (instrumentalapp.com)}
+  s.licenses    = ["MIT"]
 
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.files         = all_files
+  s.test_files    = test_files
+  s.executables   = bin_files
   s.require_paths = ["lib"]
 
   s.required_ruby_version = '>= 1.9'
@@ -20,4 +39,13 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency(%q<instrumental_agent>, [">=0.12.6"])
   s.add_runtime_dependency(%q<pidly>, [">=0.1.3"])
   s.add_development_dependency(%q<rake>, [">=0"])
+  s.add_development_dependency(%q<fpm>, [">=1.3.3"])
+  s.add_development_dependency(%q<package_cloud>, [">=0"])
+  s.add_development_dependency(%q<test-kitchen>, [">=0"])
+  s.add_development_dependency(%q<kitchen-vagrant>, [">=0"])
+  s.add_development_dependency(%q<kitchen-puppet>, [">=0"])
+  s.add_development_dependency(%q<berkshelf>, [">=0"])
+  s.add_development_dependency(%q<librarian-puppet>, [">=0"])
+  s.add_development_dependency(%q<puppet>, [">=0"])
+  s.add_development_dependency(%q<serverspec>, [">=0"])
 end

data/lib/instrumental_tools/metric_script_executor.rb

@@ -23,51 +23,55 @@ class MetricScriptExecutor
     file_stat.owned? && ((file_stat.mode & 0xFFF) ^ 0O700) == 0
   end
 
-  def run
-    process_to_output = {}
-    if can_execute_in_directory?(directory)
-      current = Dir[File.join(directory, "*")].map do |path|
-        full_path = File.expand_path(path)
-        if can_execute_file?(path)
-          stdin_r,  stdin_w  = IO.pipe
-          stdout_r, stdout_w = IO.pipe
-          stderr_r, stderr_w = IO.pipe
+  def execute_custom_script(full_path)
+    stdin_r,  stdin_w  = IO.pipe
+    stdout_r, stdout_w = IO.pipe
+    stderr_r, stderr_w = IO.pipe
 
-          previous_status, previous_time, previous_output = previous[full_path]
+    previous_status, previous_time, previous_output = previous[full_path]
 
-          stdin_w.write(previous_output || "")
-          stdin_w.close
+    stdin_w.write(previous_output || "")
+    stdin_w.close
 
 
-          cmd = [full_path, (previous_time || 0).to_i, (previous_status && previous_status.to_i)].compact.map(&:to_s)
+    cmd = [full_path, (previous_time || 0).to_i, (previous_status && previous_status.to_i)].compact.map(&:to_s)
 
-          pid = Process.spawn(*cmd,
-                              :chdir => File.dirname(full_path),
-                              :in    => stdin_r,
-                              :out   => stdout_w,
-                              :err   => stderr_w)
+    pid = Process.spawn(*cmd,
+                        :chdir => File.dirname(full_path),
+                        :in    => stdin_r,
+                        :out   => stdout_w,
+                        :err   => stderr_w)
 
-          exit_status = nil
-          exec_time   = Benchmark.realtime do
-                          pid, exit_status = Process.wait2(pid)
-                        end
+    exit_status = nil
+    exec_time   = Benchmark.realtime do
+      pid, exit_status = Process.wait2(pid)
+    end
 
-          if exec_time > 1.0
-            puts "[SLOW SCRIPT] Time to execute process #{full_path} took #{exec_time} seconds"
-          end
+    if exec_time > 1.0
+      puts "[SLOW SCRIPT] Time to execute process #{full_path} took #{exec_time} seconds"
+    end
 
-          [stdin_r, stdout_w, stderr_w].each(&:close)
+    [stdin_r, stdout_w, stderr_w].each(&:close)
 
-          output = stdout_r.read.to_s.chomp
+    output = stdout_r.read.to_s.chomp
 
-          stderr = stderr_r.read.to_s.chomp
-          unless stderr.empty?
-            puts "[STDERR] #{full_path} (PID:#{pid}) [#{Time.now.to_s}]:: #{stderr}"
-          end
+    stderr = stderr_r.read.to_s.chomp
+    unless stderr.empty?
+      puts "[STDERR] #{full_path} (PID:#{pid}) [#{Time.now.to_s}]:: #{stderr}"
+    end
+
+    [stdout_r, stderr_r].each(&:close)
 
-          [stdout_r, stderr_r].each(&:close)
+    [full_path, [exit_status, Time.now, output]]
+  end
 
-          [full_path, [exit_status, Time.now, output]]
+  def run
+    process_to_output = {}
+    if can_execute_in_directory?(directory)
+      current = Dir[File.join(directory, "*")].map do |path|
+        full_path = File.expand_path(path)
+        if can_execute_file?(path)
+          execute_custom_script(full_path)
         else
           if !File.directory?(full_path)
             uid  = Process.uid
@@ -84,7 +88,7 @@ class MetricScriptExecutor
     end
     process_to_output.flat_map do |path, (status, time, output)|
       if status && status.success?
-        prefix = File.basename(path).split(".")[0..-2].join(".").gsub(/[^a-z0-9\-\_\.]/i, "_")
+        prefix = File.basename(path).split(".")[0..-2].join(".").gsub(/[^\d\w\-\_\.]/i, "_")
         output.lines                                      # each line
           .map    { |line| line.chomp.split }             # split by whitespace
           .select { |data| (2..3).include?(data.size)  }  # and only valid name value time? pairs

data/lib/instrumental_tools/server_controller.rb

@@ -1,11 +1,13 @@
-require 'pidly'
 require 'instrumental_tools/metric_script_executor'
 require 'instrumental_tools/system_inspector'
+require 'pidly'
+require 'yaml'
 
 class ServerController < Pidly::Control
   COMMANDS = [:start, :stop, :status, :restart, :clean, :kill, :foreground]
 
   attr_accessor :run_options, :pid
+  attr_reader :current_api_key
 
   before_start do
     extra_info = if run_options[:daemon]
@@ -24,49 +26,134 @@ class ServerController < Pidly::Control
     puts 'Error encountered'
   end
 
-  def self.run(options)
-    agent = Instrumental::Agent.new(options[:api_key], :collector => [options[:collector], options[:port]].compact.join(':'))
+  def initialize(options={})
+    @run_options = options.delete(:run_options) || {}
+    super(options)
+  end
+
+  def foreground
+    run
+  end
+
+  def collector_address
+    [run_options[:collector], run_options[:port]].compact.join(':')
+  end
+
+  def user_specified_api_key
+    run_options[:api_key]
+  end
+
+  def config_file_api_key
+    if config_file_available?
+      config_contents = YAML.load(File.read(run_options[:config_file]))
+      if config_contents.is_a?(Hash)
+        config_contents['api_key']
+      end
+    end
+  rescue Exception => e
+    puts "Error loading config file %s: %s" % [run_options[:config_file], e.message]
+    nil
+  end
+
+  def configured_api_key
+    (user_specified_api_key || config_file_api_key).to_s.strip
+  end
+
+  def build_agent(key, address, enabled)
+    Instrumental::Agent.new(key, collector: address, enabled: enabled)
+  end
+
+  def set_new_agent(key, address)
+    key              = key.to_s.strip
+    @current_api_key = key
+    @agent           = build_agent(key, collector_address, key.size > 0)
+  end
+
+  def agent
+    if key_has_changed?
+      set_new_agent(configured_api_key, collector_address)
+    end
+    @agent
+  end
+
+  def report_interval
+    run_options[:report_interval]
+  end
+
+  def hostname
+    run_options[:hostname]
+  end
+
+  def script_location
+    run_options[:script_location]
+  end
+
+  def script_executor
+    @executor ||= MetricScriptExecutor.new(script_location)
+  end
+
+  def next_run_at(at_moment = Time.now.to_i)
+    (at_moment - at_moment % report_interval) + report_interval
+  end
+
+  def time_to_sleep
+    t = Time.now.to_i
+    [next_run_at(t) - t, 0].max
+  end
+
+  def config_file_available?
+    File.exists?(run_options[:config_file])
+  end
+
+  def enabled?
+    agent.enabled
+  end
+
+  def debug?
+    !!run_options[:debug]
+  end
+
+  def enable_scripts?
+    !!run_options[:enable_scripts]
+  end
+
+  def key_has_changed?
+    current_api_key != configured_api_key
+  end
+
+  def run
     puts "instrument_server version #{Instrumental::Tools::VERSION} started at #{Time.now.utc}"
-    puts "Collecting stats under the hostname: #{options[:hostname]}"
-    report_interval = options[:report_interval]
-    custom_metrics  = MetricScriptExecutor.new(options[:script_location])
+    puts "Collecting stats under the hostname: #{hostname}"
     loop do
-      t = Time.now.to_i
-      next_run_at = (t - t % report_interval) + report_interval
-      sleep [next_run_at - t, 0].max
-      inspector = SystemInspector.new
-      inspector.load_all
-      count = 0
-      inspector.gauges.each do |stat, value|
-        metric = "#{options[:hostname]}.#{stat}"
-        agent.gauge(metric, value)
-        if options[:debug]
-          puts [metric, value].join(":")
-        end
-        count += 1
-      end
-      if options[:enable_scripts]
-        custom_metrics.run.each do |(stat, value, time)|
-          metric = "#{options[:hostname]}.#{stat}"
-          agent.gauge(metric, value, time)
-          if options[:debug]
+      sleep time_to_sleep
+      if enabled?
+        inspector = SystemInspector.new
+        inspector.load_all
+        count = 0
+        inspector.gauges.each do |stat, value|
+          metric = [hostname, stat].join(".")
+          agent.gauge(metric, value)
+          if debug?
             puts [metric, value].join(":")
           end
           count += 1
         end
+        if enable_scripts?
+          script_executor.run.each do |(stat, value, time)|
+            metric = [hostname, stat].join(".")
+            agent.gauge(metric, value, time)
+            if debug?
+              puts [metric, value].join(":")
+            end
+            count += 1
+          end
+        end
+        if debug?
+          puts "Sent #{count} metrics"
+        end
       end
-      puts "Sent #{count} metrics"
     end
   end
 
-  def initialize(options={})
-    @run_options = options.delete(:run_options) || {}
-    super(options)
-  end
-
-  def foreground
-    self.class.run(run_options)
-  end
-
   alias_method :clean, :clean!
 end

data/lib/instrumental_tools/system_inspector/linux.rb

@@ -7,29 +7,60 @@ class SystemInspector
       output
     end
 
+    def self.cpu_file
+      "/proc/stat"
+    end
+
+    def self.load_file
+      "/proc/loadavg"
+    end
+
+    def self.disk_file
+      "/proc/diskstats"
+    end
+
+    def self.mount_file
+      "/proc/mounts"
+    end
+
+    def self.open_files_file
+      "/proc/sys/fs/file-nr"
+    end
+
+    def self.memory_file
+      "/proc/meminfo"
+    end
+
     def self.cpu
-      categories = [:user, :nice, :system, :idle, :iowait]
-      values     = `cat /proc/stat | grep cpu[^0-9]`.chomp.split.slice(1, 5).map { |v| v.to_f }
-      SystemInspector.memory.store(:cpu_values, values.dup)
-      if previous_values = SystemInspector.memory.retrieve(:cpu_values)
-        index = -1
-        values.collect! { |value| (previous_values[index += 1] - value).abs }
-      end
-      data   = Hash[*categories.zip(values).flatten]
-      total  = values.inject { |memo, value| memo + value }
+      agg_cpu_stat = File.read(cpu_file).lines.map { |line| line.split }.detect { |values| values.first == "cpu" }
+      output       = {}
 
-      output = {}
-      if previous_values
-        data.each do |category, value|
-          output["cpu.#{category}"] = value / total * 100
+      if agg_cpu_stat
+        categories = [:user, :nice, :system, :idle, :iowait]
+        values     = agg_cpu_stat.slice(1, 5).map { |v| v.to_f }
+        SystemInspector.memory.store(:cpu_values, values.dup)
+        if previous_values = SystemInspector.memory.retrieve(:cpu_values)
+          index = -1
+          values.collect! { |value| (previous_values[index += 1] - value).abs }
         end
+
+        data   = Hash[*categories.zip(values).flatten]
+        total  = values.inject { |memo, value| memo + value }
+
+        if previous_values
+          data.each do |category, value|
+            output["cpu.#{category}"] = value / total * 100
+          end
+        end
+
+        output["cpu.in_use"] = 100 - data[:idle] / total * 100
       end
-      output["cpu.in_use"] = 100 - data[:idle] / total * 100
+
       output
     end
 
     def self.loadavg
-      min_1, min_5, min_15 = `cat /proc/loadavg`.split
+      min_1, min_5, min_15 = File.read(load_file).split
       {
         'load.1min'  => min_1.to_f,
         'load.5min'  => min_5.to_f,
@@ -39,34 +70,41 @@ class SystemInspector
 
     def self.load_memory
       output = { :gauges => {} }
-      if SystemInspector.command_present?('free', 'memory')
+      if File.exists?(memory_file)
         output[:gauges].merge!(memory)
       end
-      if SystemInspector.command_present?('free', 'swap')
-        output[:gauges].merge!(swap)
-      end
       output
     end
 
     def self.memory
-      _, total, used, free, shared, buffers, cached = `free -k -o | grep Mem`.chomp.split
-      {
-        'memory.used_mb'      => used.to_f / 1024,
-        'memory.free_mb'      => free.to_f / 1024,
-        'memory.buffers_mb'   => buffers.to_f / 1024,
-        'memory.cached_mb'    => cached.to_f / 1024,
-        'memory.free_percent' => (free.to_f / total.to_f) * 100
-      }
-    end
+      memory_stats = Hash[File.read(memory_file).lines.map { |line| line.chomp.strip.split(/:\s+/) }.reject { |l| l.size != 2 } ]
+      total        = memory_stats["MemTotal"].to_f
+      free         = memory_stats["MemFree"].to_f
+      used         = total - free
+      buffers      = memory_stats["Buffers"].to_f
+      cached       = memory_stats["Cached"].to_f
+      swaptotal    = memory_stats["SwapTotal"].to_f
+      swapfree     = memory_stats["SwapFree"].to_f
+      swapused     = swaptotal - swapfree
+
+      stats_to_record = {
+        'memory.used_mb'      => used / 1024,
+        'memory.free_mb'      => free / 1024,
+        'memory.buffers_mb'   => buffers / 1024,
+        'memory.cached_mb'    => cached / 1024,
+        'memory.free_percent' => (free / total) * 100,
 
-    def self.swap
-      _, total, used, free = `free -k -o | grep Swap`.chomp.split
-      return {} if total.to_i == 0
-      {
-        'swap.used_mb'      => used.to_f / 1024,
-        'swap.free_mb'      => free.to_f / 1024,
-        'swap.free_percent' => (free.to_f / total.to_f) * 100
       }
+
+      if swaptotal > 0
+        stats_to_record.merge!({
+                                 'swap.used_mb'        => swapused / 1024,
+                                 'swap.free_mb'        => swapfree / 1024,
+                                 'swap.free_percent'   => (swapfree / swaptotal) * 100
+                               })
+      end
+
+      stats_to_record
     end
 
     def self.load_disks
@@ -74,7 +112,7 @@ class SystemInspector
       if SystemInspector.command_present?('df', 'disk storage')
         output[:gauges].merge!(disk_storage)
       end
-      if SystemInspector.command_present?('mount', 'disk IO')
+      if File.exists?(mount_file) && File.exists?(disk_file)
         output[:gauges].merge!(disk_io)
       end
       output
@@ -104,10 +142,10 @@ class SystemInspector
 
     def self.disk_io
       output          = {}
-      mounted_devices = `mount`.lines.grep(/^\/dev\/(\w+)/) { $1 }
-      diskstats_lines = `cat /proc/diskstats`.lines.grep(/#{mounted_devices.join('|')}/)
-      entries         = diskstats_lines.map do |line|
-                          values              = line.chomp.split
+      device_root     = "/dev/"
+      mounted_devices = File.read(mount_file).lines.map { |l| l.split.first }.select { |device| device.index(device_root) }.map { |device| File.realpath(device) }
+      diskstats_lines = File.read(disk_file).lines.map(&:split).select { |values| mounted_devices.include?(File.join(device_root, values[2])) }
+      entries         = diskstats_lines.map do |values|
                           entry               = {}
                           entry[:time]        = Time.now
                           entry[:device]      = values[2]
@@ -127,14 +165,14 @@ class SystemInspector
 
     def self.load_filesystem
       output = { :gauges => {} }
-      if SystemInspector.command_present?('sysctl', 'filesystem')
+      if File.exists?(open_files_file)
         output[:gauges].merge!(filesystem)
       end
       output
     end
 
     def self.filesystem
-      allocated, unused, max = `sysctl fs.file-nr`.split[-3..-1].map { |v| v.to_i }
+      allocated, unused, max = File.read(open_files_file).split.map(&:to_i)
       open_files             = allocated - unused
       {
         'filesystem.open_files'         => open_files,

data/lib/instrumental_tools/version.rb

@@ -1,5 +1,5 @@
 module Instrumental
   module Tools
-    VERSION = "1.0.0.rc2"
+    VERSION = "1.0.0"
   end
 end

data/puppet/.kitchen.yml

@@ -0,0 +1,26 @@
+---
+driver:
+  name: vagrant
+  provider: vmware_fusion
+
+provisioner:
+  name: puppet_apply
+  manifests_path: manifests
+  modules_path: modules
+  hiera_data_path: hieradata
+
+platforms:
+  - name: nocm_ubuntu-12.04
+    driver_plugin: vagrant
+    driver_config:
+      box: nocm_ubuntu-12.04
+      box_url: https://atlas.hashicorp.com/puppetlabs/boxes/ubuntu-12.04-64-nocm/versions/1.0.1/providers/vmware_fusion.box
+  - name: nocm_centos-6.6
+    driver_plugin: vagrant
+    driver_config:
+      box: nocm_centos-6.6
+      box_url: https://atlas.hashicorp.com/puppetlabs/boxes/centos-6.6-64-nocm/versions/1.0.1/providers/vmware_fusion.box
+
+suites:
+  - name: default
+    manifest: site.pp