server_metrics 0.0.2

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea
+.ruby-version

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in server_metrics.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Andre Lewis
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,34 @@
+# SystemMetrics
+
+Collects key metrics on CPU, disks, memory, network interfaces, and processes.
+
+## Use
+
+    $ gem install server_metrics
+
+    require 'server_metrics'
+    cpu=ServerMetrics::Cpu.new
+    cpu.run
+    pp cpu.data.inspect
+
+    # same with:
+
+    ServerMetrics::Disk.new
+    ServerMetrics::Memory.new
+    ServerMetrics::Network.new
+
+    # also see:
+
+    ServerMetrics::SystemInfo.to_hash
+
+## Creating a new collector
+
+Inherit from either Collector (if you're generating just one set of data), or MultiCollector (if you will be generating N
+sets of data -- for example if you're monitoring disks or network interfaces).
+
+## TODOs
+
+* better mac compatibility
+* more test coverage
+* rethink module hierarchy
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/test*.rb']
+  t.verbose = true
+end
+
+task :default => :test

data/example_processes.rb

@@ -0,0 +1,20 @@
+require "rubygems"
+require File.dirname(__FILE__)+ "/lib/server_metrics"
+require "pry"
+require "awesome_print"
+
+
+p = ServerMetrics::Processes.new(1)
+puts "stating ..."
+p.run
+sleep 1
+ap p.run
+
+#p.get_overall_cpu
+#ServerMetrics::ProcessList.add_cpu_time(ServerMetrics::ProcessList.group)
+#puts "sleeping ..."
+#sleep 1
+#puts "#### overall"
+#ap p.get_overall_cpu
+#puts "#### individual processes"
+#ap ServerMetrics::ProcessList.add_cpu_time(ServerMetrics::ProcessList.group)

data/example_usage.rb

@@ -0,0 +1,43 @@
+require "rubygems"
+require File.dirname(__FILE__)+ "/lib/server_metrics"
+require "pry"
+
+class Harness
+  attr_accessor :num_runs, :latest_run
+
+  def initialize
+    @num_runs=0
+
+    @collectors={:disks => ServerMetrics::Disk.new(), :cpu => ServerMetrics::Cpu.new(), :memory => ServerMetrics::Memory.new(), :network => ServerMetrics::Network.new(), :processes=>ServerMetrics::Processes.new(ServerMetrics::SystemInfo.num_processors)}
+
+    @system_info = ServerMetrics::SystemInfo.to_h
+  end
+
+  def run
+    collector_res={}
+    @collectors.each_pair do |name, collector|
+      collector_res[name] = collector.run
+    end
+
+    @latest_run = collector_res.merge(:system_info => @system_info)
+
+    @num_runs +=1
+  end
+end
+
+harness = Harness.new
+
+harness.run
+sleep 1
+harness.run
+pp harness.latest_run
+
+#puts "starting"
+#while(true) do
+#  harness.run
+#  puts "running at #{Time.now}"
+#  File.open("server_metrics.json","w") do |f|
+#    f.puts harness.latest_run
+#  end
+#  sleep 15
+#end

data/lib/server_metrics/collector.rb

@@ -0,0 +1,147 @@
+
+# The base class for SystemMetrics collectors.
+#
+# Some collects inherit directly from Collector, and some inherit from MultiCollector.
+# The difference: if you're collecting for an arbitrary number of instances (say, disks), use MultiCollector.
+# Otherwise, use Collector.
+#
+# Relative to Scout's plugins, Collectors have a few differences:
+#
+# 1. simplified: no options parsing. simpler interface to reporting and memory (these methods only take a hash)
+# 2. intended to persist in memory: a collector maintains its own memory. Reuse the same instance as many times as needed.
+#    If you need to persist to disk, use the to_hash and from_hash methods.
+#
+module ServerMetrics
+  class Collector
+    attr_reader :collector_id
+    attr_accessor :data, :error
+
+    def initialize(options={})
+      @options = options
+      @data={}
+      @memory={}
+      @collector_id = self.class.name+'-'+@options.to_a.sort_by { |a| a.first }.flatten.join('-')
+      @error=nil
+    end
+
+    def option(name)
+      @options[name] || @options[name.is_a?(String) ? name.to_sym : String(name)]
+    end
+
+    def run
+      @data={}
+      build_report
+      @data
+    end
+
+    def report(hash)
+      @data.merge!(hash)
+    end
+
+    #   memory(:no_track)
+    #   memory.delete(:no_track)
+    #   memory.clear
+    #
+    def memory(name = nil)
+      if name.nil?
+        @memory
+      else
+        @memory[name] || @memory[name.is_a?(String) ? name.to_sym : String(name)]
+      end
+    end
+
+
+    #   remember(name1: value1, name2: value2)
+    #
+    def remember(hash)
+      @memory.merge!(hash)
+    end
+
+    #   counter(:rkbps, stats['rsect'] / 2, :per => :second)
+    #   counter(:rpm, request_counter, :per => :minute)
+    #   counter(:swap_ins, vmstat['pswpin'], :per => :second, :round => true)
+    #
+    def counter(name, value, options = {}, &block)
+      current_time = Time.now
+
+      if data = memory("_counter_#{name}")
+        last_time, last_value = data[:time], data[:value]
+        elapsed_seconds = current_time - last_time
+
+        # We won't log it if the value has wrapped or enough time hasn't
+        # elapsed
+        if value >= last_value && elapsed_seconds >= 1
+          if block
+            result = block.call(last_value, value)
+          else
+            result = value - last_value
+          end
+
+          case options[:per]
+            when :second, 'second'
+              result = result / elapsed_seconds.to_f
+            when :minute, 'minute'
+              result = result / elapsed_seconds.to_f * 60.0
+            else
+              raise "Unknown option for ':per': #{options[:per].inspect}"
+          end
+
+          if options[:round]
+            result = (result * (10 ** options[:round])).round / (10 ** options[:round]).to_f
+          end
+
+          report(name => result)
+        end
+      end
+
+      remember("_counter_#{name}" => {:time => current_time, :value => value})
+    end
+
+    # Convert strings containing 'T,G,M,or K' to MB. The result is a float only -- units are NOT returned
+    def convert_to_mb(value)
+      value = if value =~ /G/i
+                value.to_f*1024.0
+              elsif value =~ /M/i
+                value.to_f
+              elsif value =~ /K/i
+                (value.to_f/1024.0)
+              elsif value =~ /T/i
+                (value.to_f*1024.0*1024.0)
+              else
+                value.to_f
+              end
+      ("%.1f" % [value]).to_f
+    end
+
+    #
+    def normalize_key(key)
+      (key.is_a?(String) ? key : key.to_s).downcase.gsub(" ", "_").gsub("%", "percent").to_sym
+    end
+
+    # returns a hash you can serialize and store on disk, or just hold onto and re-instantiate the collector later.
+    # Why you'd need to do this: to persist the memory (including counters) of a plugin instance.
+    #
+    # Collector.from_hash(h) is the flipside of this: Collector.from_hash(plugin.to_hash) gets you essentially the same instance
+    #
+    def to_hash
+      {:options => @options, :memory => @memory, :data => @data, :plugin_id => @plugin_id}
+    end
+
+    # see to_hash. The hash should contain :options and :memory keys
+    def self.from_hash(hash)
+      c=Collector.new(hash[:options])
+      c.instance_variable_set('@memory', hash[:memory])
+      c.instance_variable_set('@data', hash[:data])
+      c
+    end
+
+    def linux?
+      RbConfig::CONFIG['target_os'] == 'linux'
+    end
+
+    def osx?
+      RbConfig::CONFIG['target_os'] == 'darwin'
+    end
+
+  end
+end

data/lib/server_metrics/collectors/cpu.rb

@@ -0,0 +1,137 @@
+require "time"
+
+class ServerMetrics::Cpu < ServerMetrics::Collector
+
+  # Raised by #CpuStats when an error reading /proc/stats.
+  class ProcStatError < Exception
+  end
+
+  def build_report
+    begin
+      stats = CpuStats.fetch
+
+      if previous = memory(:cpu_stats)
+        previous_stats = CpuStats.from_hash(previous)
+
+        report stats.diff(previous_stats)
+      end
+
+      remember(:cpu_stats => stats.to_h)
+    rescue ProcStatError
+      @error = "could not retrieve CPU stats from /proc/stat"
+    end
+
+    ENV['LANG'] = 'C' # forcing english for parsing
+    uptime_output = `uptime`
+    matches = uptime_output.match(/load averages?: ([\d.]+),? ([\d.]+),? ([\d.]+)\Z/)
+
+    report("Last minute" => matches[1].to_f,
+           "Last five minutes" => matches[2].to_f,
+           "Last fifteen minutes" => matches[3].to_f)
+  end
+
+  # Helper class
+  class CpuStats
+    attr_accessor :user, :system, :idle, :iowait, :interrupts, :procs_running, :procs_blocked, :time, :steal
+
+    def self.fetch
+      output = `cat /proc/stat 2>&1`
+
+      if $? and !$?.success?
+        raise ProcStatError, output
+      end
+
+      data = output.split(/\n/).collect { |line| line.split }
+
+      cpu_stats = CpuStats.new
+
+      if cpu = data.detect { |line| line[0] == 'cpu' }
+        cpu_stats.user, nice, cpu_stats.system, cpu_stats.idle, cpu_stats.iowait,
+            hardirq, softirq, cpu_stats.steal = *cpu[1..-1].collect { |c| c.to_i }
+        cpu_stats.user += nice
+        cpu_stats.system += hardirq + softirq
+      end
+
+      if interrupts = data.detect { |line| line[0] == 'intr' }
+        cpu_stats.interrupts, _ = *interrupts[1..-1].collect { |c| c.to_i }
+      end
+
+      if procs_running = data.detect { |line| line[0] == 'procs_running' }
+        cpu_stats.procs_running, _ = *procs_running[1..-1].collect { |c| c.to_i }
+      end
+
+      if procs_blocked = data.detect { |line| line[0] == 'procs_blocked' }
+        cpu_stats.procs_blocked, _ = *procs_blocked[1..-1].collect { |c| c.to_i }
+      end
+
+      cpu_stats
+    end
+
+    def self.from_hash(h)
+      cpu_stats= CpuStats.new
+      hash = {}
+      h.each { |k, v| hash[k.to_sym] = v }
+
+      if time = hash.delete(:time)
+        cpu_stats.time = Time.parse(time) rescue time
+      end
+
+      hash.each do |k, v|
+        cpu_stats.send("#{k}=", v) if cpu_stats.respond_to?("#{k}=")
+      end
+      cpu_stats
+    end
+
+    def initialize
+      self.time = Time.now
+    end
+
+    def diff(other)
+      diff_user = user - other.user
+      diff_system = system - other.system
+      diff_idle = idle - other.idle
+      diff_iowait = iowait - other.iowait
+
+      div = diff_user + diff_system + diff_idle + diff_iowait
+
+      if steal && other.steal && steal > 0
+        diff_steal = steal - other.steal
+        div += diff_steal
+      end
+
+      divo2 = div / 2
+
+      results = {
+          "User" => (100.0 * diff_user + divo2) / div,
+          "System" => (100.0 * diff_system + divo2) / div,
+          "Idle" => (100.0 * diff_idle + divo2) / div,
+          "IO wait" => (100.0 * diff_iowait + divo2) / div,
+          "Procs running" => self.procs_running,
+          "Procs blocked" => self.procs_blocked
+      }
+
+      if diff_steal && steal > 0
+        results["Steal"] = (100.0 * diff_steal + divo2) / div
+      end
+
+      if self.time && other.time
+        diff_in_seconds = self.time.to_f - other.time.to_f
+
+        results["Interrupts"] = (self.interrupts.to_f - other.interrupts.to_f) / diff_in_seconds
+      end
+
+      results
+    end
+
+    def to_h
+      {
+          :user => user, :system => system, :idle => idle, :iowait => iowait,
+          :interrupts => interrupts, :procs_running => procs_running,
+          :procs_blocked => procs_blocked, :time => Time.now.to_s,
+          :steal => steal
+      }
+    end
+
+  end
+end
+

data/lib/server_metrics/collectors/disk.rb

@@ -0,0 +1,106 @@
+# Collects Disk metrics on eligible filesystems. Reports a hash of hashes, with the first hash keyed by device name.
+#
+# TODO: Currently, this reports on devices that begins with /dev as listed by `mount`. Revisit this.
+# TODO: relies on /proc/diskstats, so not mac compatible. Figure out mac compatibility
+#
+class ServerMetrics::Disk < ServerMetrics::MultiCollector
+
+  def build_report
+    @df_output = `df -h`.split("\n")
+    @devices = `mount`.split("\n").grep(/^\/dev/).map{|l|l.split.first} # any device that starts with /dev
+
+    @devices.each do |device|
+      get_sizes(device) # does its own reporting
+      get_stats(device) if linux? # does its own reporting
+    end
+  end
+
+  # called from build_report for each device
+  def get_sizes(device)
+    ENV['LANG'] = 'C' # forcing English for parsing
+
+    header_line=@df_output.first
+    num_columns = header_line.include?("iused") ? 9 : 6 # Mac has extra columns
+    headers = header_line.split(/\s+/,num_columns)
+    parsed_lines=[] # Each line will look like {"%iused" => "38%","Avail" => "289Gi", "Capacity=> "38%", "Filesystem"=> "/dev/disk0s2","Mounted => "/", "Size" => "465Gi", "Used" => "176Gi", "ifree" => "75812051", "iused"  => "46116178"}
+
+    @df_output[1..@df_output.size-2].each do |line|
+      values=line.split(/\s+/,num_columns)
+      parsed_lines<<Hash[headers.zip(values)]
+    end
+
+    # select the right line
+    hash = parsed_lines.select{|l| l["Filesystem"] == device}.first
+    result = {}
+    hash.each_pair do |key,value|
+      #key=normalize_key(key) # downcase, make a symbol, etc
+      value = convert_to_mb(value) if [:avail,:capacity,:size,:used, :usepercent].include?(key)
+      result[key]=value
+    end
+
+    report(device, result)
+  end
+
+  # called from build_report for each device
+  def get_stats(device)
+    stats = iostat(device)
+
+    if stats
+      counter(device, "RPS",   stats['rio'],        :per => :second)
+      counter(device, "WPS",   stats['wio'],        :per => :second)
+      counter(device, "Kb RPS", stats['rsect'] / 2,  :per => :second)
+      counter(device, "Kb WPS", stats['wsect'] / 2,  :per => :second)
+      counter(device, "Utilization",  stats['use'] / 10.0, :per => :second)
+      # Not 100% sure that average queue length is present on all distros.
+      if stats['aveq']
+        counter(device, "Average queue length",  stats['aveq'], :per => :second)
+      end
+
+      if old = memory(device, "stats")
+        ios = (stats['rio'] - old['rio']) + (stats['wio']  - old['wio'])
+
+        if ios > 0
+          await = ((stats['ruse'] - old['ruse']) + (stats['wuse'] - old['wuse'])) / ios.to_f
+
+          report(device, "Await" => await)
+        end
+      end
+
+      remember(device, "stats" => stats)
+    end
+  end
+
+  private
+  COLUMNS = %w(major minor name rio rmerge rsect ruse wio wmerge wsect wuse running use aveq)
+
+  # Returns the /proc/diskstats line associated with device name +dev+. Logic:
+  #
+  # * If an exact match of the specified device is found, returns it.
+  # * If there isn't an exact match but there are /proc/diskstats lines that are included in +dev+,
+  #   returns the first matching line. This is needed as the mount output used to find the default device doesn't always
+  #   match /proc/diskstats output.
+  # * If there are no matches but an LVM is used, returns the line matching "dm-0".
+  def iostat(dev)
+    # if a LVM is used, `mount` output doesn't map to `/diskstats`. In this case, use dm-0 as the default device.
+    lvm = nil
+    retried = false
+    possible_devices = []
+    begin
+      %x(cat /proc/diskstats).split(/\n/).each do |line|
+        entry = Hash[*COLUMNS.zip(line.strip.split(/\s+/).collect { |v| Integer(v) rescue v }).flatten]
+        possible_devices << entry if dev.include?(entry['name'])
+        lvm = entry if (@default_device_used and 'dm-0'.include?(entry['name']))
+      end
+    rescue Errno::EPIPE
+      if retried
+        raise
+      else
+        retried = true
+        retry
+      end
+    end
+    found_device = possible_devices.find { |entry| dev == entry['name'] } || possible_devices.first
+    return found_device || lvm
+  end
+
+end