server_metrics 1.1.1 → 1.2.0.pre

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+# 1.2.0
+
+Performance related:
+
+* Using `Dir#glob` vs. `Dir#foreach`
+* Using `File#read` vs. `cat FILE`. 
+* Removing commented out and unused elements from `SysLite::ProcTable`
+* Removed method_missing logic in `Processes::Process` to access the `ProcTableStruct`
+* `Disk` added caching logic via a `@option[:ttl]` to execute slow system calls at a lower interval (ex: ev 60 seconds)
+* `String#lines#to_a` vs. `String#split('\n')` (faster)
+
 ## 1.1.1
 
 * Handling Infinite and NaN Process CPU Usage.

data/lib/server_metrics/collectors/cpu.rb

@@ -21,16 +21,22 @@ class ServerMetrics::Cpu < ServerMetrics::Collector
     rescue ProcStatError
       @error = "could not retrieve CPU stats from /proc/stat"
     end
-
-    ENV['LANG'] = 'C' # forcing english for parsing
-    uptime_output = `uptime`
-    matches = uptime_output.match(/load averages?: ([\d.]+),? ([\d.]+),? ([\d.]+)\Z/)
-
-    number_of_processors = ServerMetrics::SystemInfo.num_processors
-
-    report(:last_minute => matches[1].to_f / number_of_processors,
-           :last_five_minutes => matches[2].to_f / number_of_processors,
-           :last_fifteen_minutes => matches[3].to_f / number_of_processors)
+    
+    # This requires a system call, which is slow. `scout_realtime` doesn't display server load, so this
+    # option allows `scout_realtime` to not collect load averages.
+    if !@options[:skip_load]
+      ENV['LANG'] = 'C' # forcing english for parsing
+      uptime_output = `uptime`
+      matches = uptime_output.match(/load averages?: ([\d.]+),? ([\d.]+),? ([\d.]+)\Z/)
+
+      report(:last_minute => matches[1].to_f / num_processors,
+             :last_five_minutes => matches[2].to_f / num_processors,
+             :last_fifteen_minutes => matches[3].to_f / num_processors)
+    end
+  end
+  
+  def num_processors
+    @num_processors ||= ServerMetrics::SystemInfo.num_processors  
   end
 
   # Helper class
@@ -38,13 +44,16 @@ class ServerMetrics::Cpu < ServerMetrics::Collector
     attr_accessor :user, :system, :idle, :iowait, :interrupts, :procs_running, :procs_blocked, :time, :steal
 
     def self.fetch
-      output = `cat /proc/stat 2>&1`
-
-      if $? and !$?.success?
-        raise ProcStatError, output
+      output = nil
+      begin
+        output = File.read("/proc/stat")
+      rescue Errno::ENOENT
+        # No such file or directory - /proc/stat
+        # /proc/stat doesn't exist on this system.
+        raise ProcStatError
       end
 
-      data = output.split(/\n/).collect { |line| line.split }
+      data = output.lines.collect { |line| line.split }
 
       cpu_stats = CpuStats.new
 

data/lib/server_metrics/collectors/disk.rb

@@ -7,23 +7,42 @@ class ServerMetrics::Disk < ServerMetrics::MultiCollector
 
   def build_report
     ENV['LANG'] = 'C' # forcing English for parsing
-    @df_output = `df -Pkh`.split("\n")
-    @devices = `mount`.split("\n").grep(/^\/dev/).map{|l|l.split.first} # any device that starts with /dev
-    @disk_stats = `cat /proc/diskstats`.split("\n")
+    
+    @disk_stats = File.read("/proc/diskstats").lines.to_a
 
-    @devices.each do |device|
+    devices.each do |device|
       get_sizes(device) # does its own reporting
       get_stats(device) if linux? # does its own reporting
     end
   end
+  
+  # System calls are slow. Read once every minute and not on every innvocation. 
+  def df_output
+    if @last_df_output.nil? or @last_df_output < (Time.now-@options[:ttl].to_i*60)
+      @last_df_output = Time.now
+      @df_output = `df -Pkh`.lines.to_a
+    else
+      @df_output
+    end
+  end
+  
+  # System calls are slow. Read once every minute and not on every innvocation. 
+  def devices
+    if @devices.nil? or @last_devices_output < (Time.now-@options[:ttl].to_i*60)
+      @last_devices_output = Time.now
+      @devices = `mount`.split("\n").grep(/^\/dev/).map{|l|l.split.first} # any device that starts with /dev    
+    else
+      @devices
+    end
+  end
 
   # called from build_report for each device
   def get_sizes(device)
-    header_line=@df_output.first
+    header_line=df_output.first
     headers = header_line.split(/\s+/,6) # limit to 6 columns - last column is "mounted on"
     parsed_lines=[] # Each line will look like {"%iused" => "38%","Avail" => "289Gi", "Capacity=> "38%", "Filesystem"=> "/dev/disk0s2","Mounted => "/", "Size" => "465Gi", "Used" => "176Gi", "ifree" => "75812051", "iused"  => "46116178"}
 
-    @df_output[1..@df_output.size-1].each do |line|
+    df_output[1..df_output.size-1].each do |line|
       values=line.split(/\s+/,6)
       parsed_lines<<Hash[headers.zip(values)]
     end

data/lib/server_metrics/collectors/memory.rb

@@ -17,7 +17,7 @@ class ServerMetrics::Memory < ServerMetrics::Collector
 
   def linux_memory
     mem_info = {}
-    `cat /proc/meminfo`.each_line do |line|
+    File.read("/proc/meminfo").each_line do |line|
       _, key, value = *line.match(/^(\w+):\s+(\d+)\s/)
       mem_info[key] = value.to_i
     end

data/lib/server_metrics/collectors/network.rb

@@ -5,7 +5,7 @@ class ServerMetrics::Network < ServerMetrics::MultiCollector
   def build_report
 
     if linux?
-      lines = %x(cat /proc/net/dev).split("\n")[2..-1]
+      lines = File.read("/proc/net/dev").lines.to_a[2..-1]
       interfaces = []
       lines.each do |line|
         iface, rest = line.split(':', 2).collect { |e| e.strip }

data/lib/server_metrics/collectors/processes.rb

@@ -195,9 +195,33 @@ class ServerMetrics::Processes
       (utime || 0) + (stime || 0)  # utime and stime aren't available on mac. Result is %cpu is 0 on mac.
     end
     
-    # delegate everything else to ProcTable::Struct
-    def method_missing(sym, *args, &block)
-      @pts.send sym, *args, &block
+    # For better performance, not using #method_missing to just pass these off to ProcTable::Struct.
+    def pid
+      @pts.pid
+    end
+    
+    def comm
+      @comm ||= @pts.comm
+    end
+    
+    def cmdline
+      @pts.cmdline
+    end
+    
+    def ppid
+      @pts.ppid
+    end
+    
+    def utime
+      @pts.utime
+    end
+    
+    def stime
+      @pts.stime
+    end
+    
+    def rss
+      @pts.rss
     end
   end
 end

data/lib/server_metrics/lib/proctable_lite.rb

@@ -17,9 +17,6 @@ module SysLite
     VERSION = '0.9.3'
 
     private
-
-    # @mem_total = IO.read("/proc/meminfo")[/MemTotal.*/].split[1].to_i * 1024 rescue nil
-    # @boot_time = IO.read("/proc/stat")[/btime.*/].split.last.to_i rescue nil
     
     # Handles a special case on Ubuntu - kthreadd generates many children (200+). 
     # These are aggregated together and reported as a single process, kthreadd.
@@ -39,57 +36,17 @@ module SysLite
     @fields = [
         'cmdline',     # Complete command line
         'cwd',         # Current working directory
-        'environ',     # Environment
         'exe',         # Actual pathname of the executed command
-        'fd',          # File descriptors open by process
-        'root',        # Root directory of process
         'pid',         # Process ID
         'comm',        # Filename of executable
-        'state',       # Single character state abbreviation
         'ppid',        # Parent process ID
-        'pgrp',        # Process group
-        'session',     # Session ID
-        'tty_nr',      # TTY (terminal) associated with the process
-        'tpgid',       # Group ID of the TTY
-        'flags',       # Kernel flags
-        'minflt',      # Number of minor faults
-        'cminflt',     # Number of minor faults of waited-for children
-        'majflt',      # Number of major faults
-        'cmajflt',     # Number of major faults of waited-for children
         'utime',       # Number of user mode jiffies
         'stime',       # Number of kernel mode jiffies
         'cutime',      # Number of children's user mode jiffies
         'cstime',      # Number of children's kernel mode jiffies
-        'priority',    # Nice value plus 15
-        'nice',        # Nice value
-        'itrealvalue', # Time in jiffies before next SIGALRM
-        'starttime',   # Time in jiffies since system boot
         'vsize',       # Virtual memory size in bytes
         'rss',         # Resident set size
-        'rlim',        # Current limit on the rss in bytes
-        'startcode',   # Address above which program text can run
-        'endcode',     # Address below which program text can run
-        'startstack',  # Address of the startstack
-        'kstkesp',     # Kernel stack page address
-        'kstkeip',     # Kernel instruction pointer
-        'signal',      # Bitmap of pending signals
-        'blocked',     # Bitmap of blocked signals
-        'sigignore',   # Bitmap of ignored signals
-        'sigcatch',    # Bitmap of caught signals
-        'wchan',       # Channel in which the process is waiting
-        'nswap',       # Number of pages swapped
-        'cnswap',      # Cumulative nswap for child processes
-        'exit_signal', # Signal to be sent to parent when process dies
-        'processor',   # CPU number last executed on
-        'rt_priority', # Real time scheduling priority
-        'policy',      # Scheduling policy
         'name',        # Process name
-        'uid',         # Real user ID
-        'euid',        # Effective user ID
-        'gid',         # Real group ID
-        'egid',        # Effective group ID
-        'pctcpu',     # Percent of CPU usage (custom field)
-        'pctmem'      # Percent of Memory usage (custom field)
     ]
 
     public
@@ -123,57 +80,13 @@ module SysLite
       array  = block_given? ? nil : []
       struct = nil
       raise TypeError unless pid.is_a?(Fixnum) if pid
-
-      Dir.foreach("/proc"){ |file|
-        next if file =~ /\D/ # Skip non-numeric directories
+      
+      Dir.chdir("/proc")
+      Dir.glob("[0-9]*").each do |file|
         next unless file.to_i == pid if pid
 
         struct = ProcTableStruct.new
 
-        # Get /proc/<pid>/cmdline information. Strip out embedded nulls.
-        # begin
-        #   data = IO.read("/proc/#{file}/cmdline").tr("\000", ' ').strip
-        #   struct.cmdline = data
-        # rescue
-        #   next # Process terminated, on to the next process
-        # end
-
-        # Get /proc/<pid>/cwd information
-        # struct.cwd = File.readlink("/proc/#{file}/cwd") rescue nil
-
-        # Get /proc/<pid>/environ information. Environment information
-        # is represented as a Hash, with the environment variable as the
-        # key and its value as the hash value.
-        struct.environ = {}
-
-        # begin
-        #           IO.read("/proc/#{file}/environ").split("\0").each{ |str|
-        #             key, value = str.split('=')
-        #             struct.environ[key] = value
-        #           }
-        #         rescue Errno::EACCES, Errno::ESRCH, Errno::ENOENT
-        #           # Ignore and move on.
-        #         end
-
-        # Get /proc/<pid>/exe information
-        # struct.exe = File.readlink("/proc/#{file}/exe") rescue nil
-
-        # Get /proc/<pid>/fd information. File descriptor information
-        # is represented as a Hash, with the fd as the key, and its
-        # symlink as the value.
-        struct.fd = {}
-
-        # begin
-        #           Dir["/proc/#{file}/fd/*"].each do |fd|
-        #             struct.fd[File.basename(fd)] = File.readlink(fd) rescue nil
-        #           end
-        #         rescue
-        #           # Ignore and move on
-        #         end
-
-        # Get /proc/<pid>/root information
-        # struct.root = File.readlink("/proc/#{file}/root") rescue nil
-
         # Get /proc/<pid>/stat information
         stat = IO.read("/proc/#{file}/stat") rescue next
 
@@ -188,66 +101,10 @@ module SysLite
         struct.pid         = stat[0].to_i
         # Remove parens. Note this could be overwritten in #get_comm_group_name.
         struct.comm        = stat[1].tr('()','')
-        # struct.state       = stat[2]
         struct.ppid        = stat[3].to_i
-        #        struct.pgrp        = stat[4].to_i
-        #        struct.session     = stat[5].to_i
-        #        struct.tty_nr      = stat[6].to_i
-        #        struct.tpgid       = stat[7].to_i
-        #        struct.flags       = stat[8].to_i
-        #        struct.minflt      = stat[9].to_i
-        #        struct.cminflt     = stat[10].to_i
-        #        struct.majflt      = stat[11].to_i
-        #        struct.cmajflt     = stat[12].to_i
         struct.utime       = stat[13].to_i
         struct.stime       = stat[14].to_i
-        # struct.cutime      = stat[15].to_i
-        #        struct.cstime      = stat[16].to_i
-        #        struct.priority    = stat[17].to_i
-        #        struct.nice        = stat[18].to_i
-        # Skip 19
-        # struct.itrealvalue = stat[20].to_i
-        #         struct.starttime   = stat[21].to_i
-        #         struct.vsize       = stat[22].to_i
         struct.rss         = stat[23].to_i
-        # struct.rlim        = stat[24].to_i
-        #         struct.startcode   = stat[25].to_i
-        #         struct.endcode     = stat[26].to_i
-        #         struct.startstack  = stat[27].to_i
-        #         struct.kstkesp     = stat[28].to_i
-        #         struct.kstkeip     = stat[29].to_i
-        #         struct.signal      = stat[30].to_i
-        #         struct.blocked     = stat[31].to_i
-        #         struct.sigignore   = stat[32].to_i
-        #         struct.sigcatch    = stat[33].to_i
-        #         struct.wchan       = stat[34].to_i
-        #         struct.nswap       = stat[35].to_i
-        #         struct.cnswap      = stat[36].to_i
-        #         struct.exit_signal = stat[37].to_i
-        #         struct.processor   = stat[38].to_i
-        #         struct.rt_priority = stat[39].to_i
-        #         struct.policy      = stat[40].to_i
-
-        # Get /proc/<pid>/status information (name, uid, euid, gid, egid)
-        # IO.foreach("/proc/#{file}/status") do |line|
-        #           case line
-        #             when /Name:\s*?(\w+)/
-        #               struct.name = $1
-        #             when /Uid:\s*?(\d+)\s*?(\d+)/
-        #               struct.uid  = $1.to_i
-        #               struct.euid = $2.to_i
-        #             when /Gid:\s*?(\d+)\s*?(\d+)/
-        #               struct.gid  = $1.to_i
-        #               struct.egid = $2.to_i
-        #           end
-        #         end
-
-        # If cmdline is empty use comm instead
-        # struct.cmdline = struct.comm if struct.cmdline.empty?
-
-        # Manually calculate CPU and memory usage
-        # struct.pctcpu = get_pctcpu(struct.utime, struct.starttime)
-        # struct.pctmem = get_pctmem(struct.rss)
         
         # don't report kthreadd chidren individually - aggregate into the parent.
         if kthreadd_child?(struct.ppid)
@@ -267,7 +124,7 @@ module SysLite
         else
           array << struct
         end
-      }
+      end # Dir.glob
 
       if pid
         struct

data/lib/server_metrics/system_info.rb

@@ -20,7 +20,7 @@ module ServerMetrics
       if os =~ /(darwin|freebsd)/
         `sysctl -n hw.ncpu`.to_i
       elsif os =~ /linux/
-        lines = `cat /proc/cpuinfo`.split("\n")
+        lines = File.read("/proc/cpuinfo").lines.to_a
         lines.grep(/^processor\s*:/i).size
       end
     rescue

data/lib/server_metrics/version.rb

@@ -1,3 +1,3 @@
 module ServerMetrics
-  VERSION = "1.1.1"
+  VERSION = "1.2.0.pre"
 end

data/test/performance/prof.rb

@@ -0,0 +1,44 @@
+# Generates ruby-prof output based on the config options used by `scout_realtime` optimized for performance.
+# ruby prof.rb
+require "rubygems"
+require "server_metrics"
+require "ruby-prof"
+
+class Harness
+  attr_accessor :num_runs, :latest_run
+
+  def initialize
+    @num_runs=0
+
+    @collectors={:disks => ServerMetrics::Disk.new(:ttl => 60), :cpu => ServerMetrics::Cpu.new(:skip_load => true), :memory => ServerMetrics::Memory.new(), :network => ServerMetrics::Network.new(), :processes=>ServerMetrics::Processes.new()}
+    @system_info = ServerMetrics::SystemInfo.to_h
+  end
+
+  def run
+    collector_res={}
+    @collectors.each_pair do |name, collector|
+      collector_res[name] = collector.run
+    end
+
+    @latest_run = collector_res.merge(:system_info => @system_info)
+
+    @num_runs +=1
+  end
+end
+
+harness = Harness.new
+
+harness.run
+sleep 1
+RUNS = 300
+result = RubyProf.profile do
+  i=0
+  while i < RUNS do
+    i +=1
+    harness.run
+  end
+end
+
+# Print a graph profile to text
+printer = RubyProf::GraphPrinter.new(result)
+printer.print(STDOUT, {})

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: server_metrics
 version: !ruby/object:Gem::Version
-  version: 1.1.1
-  prerelease: 
+  version: 1.2.0.pre
+  prerelease: 6
 platform: ruby
 authors:
 - Andre Lewis
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-21 00:00:00.000000000 Z
+date: 2014-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -140,6 +140,7 @@ files:
 - test/fixtures/disk.txt
 - test/fixtures/memory.txt
 - test/fixtures/network.txt
+- test/performance/prof.rb
 - test/test.rb
 - test/test_basics.rb
 - test/test_helper.rb
@@ -157,18 +158,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 4512934295163218647
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
-      segments:
-      - 0
-      hash: 4512934295163218647
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -180,7 +175,9 @@ test_files:
 - test/fixtures/disk.txt
 - test/fixtures/memory.txt
 - test/fixtures/network.txt
+- test/performance/prof.rb
 - test/test.rb
 - test/test_basics.rb
 - test/test_helper.rb
 - test/test_with_fixtures.rb
+has_rdoc: