instrumental_tools 0.1.0

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+/pkg

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/README.md

@@ -0,0 +1,27 @@
+# Instrumental Tools
+
+Collection of tools for use with the Instrumental (http://www.instrumentalapp.com/) app
+
+## instrument_server
+
+Use to collect various monitoring statistics of a server. Execute with:
+
+```sh
+instrument_server [INSTRUMENTAL_API_KEY]
+```
+
+Linux note: Install iostat (part of the sysstat package) in order to collect disk I/O metrics.
+
+## instrumental
+
+Output text graphs of the different metrics in your project.
+
+See all options with: `instrumental --help`
+
+## gitstrumental
+
+Collect statistics on commit counts in a given git repo.  Execute in the repo directory with:
+
+```sh
+gitstrumental [INSTRUMENTAL_API_KEY]
+```

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/bin/gitstrumental

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+gem 'instrumental_agent'
+require 'instrumental_agent'
+
+token = ENV["INSTRUMENTAL_TOKEN"] || ARGV[0]
+
+if !token 
+  puts 'Usage: gitstrumental API_TOKEN'
+  exit 1
+end
+
+lines = `git log --format="REC %at %h '%cn'" --numstat`.chomp.split(/\n+/).reject(&:empty?)
+branch = File.basename(`git symbolic-ref HEAD`.chomp).gsub(/[^a-z0-9]/i, "_")
+if fetch_url = `git remote show origin`.chomp.split(/\n+/).grep(/Fetch URL:/).first
+  repo = File.basename(fetch_url.split("/").last, ".git")
+else
+  repo = File.basename(Dir.pwd, ".git")
+end
+prolog = [repo, branch].join(".")
+I = Instrumental::Agent.new(token)
+curstat = {}
+lines.each do |line|
+  if line =~ /^REC/
+    if !curstat.empty?
+      name, ts, changes = curstat[:name], curstat[:timestamp], curstat[:changes]
+      I.increment("git.#{prolog}.commits.#{name}", 1, ts)
+
+      adds, deletes = changes.reduce do |prev, pair|
+        [prev[0] + pair[0], prev[1] + pair[1]]
+      end
+      I.increment("git.#{prolog}.commits.#{name}.add", adds, ts)
+      I.increment("git.#{prolog}.commits.#{name}.delete", deletes, ts)
+      I.increment("git.#{prolog}.num_commits", 1, ts)
+    end
+    rec, ts, hash, *name_parts = line.split(" ")
+    name = name_parts.join(" ").gsub(/^'([^']+)'$/, "\\1").gsub(/[^a-z0-9]/i, "_")
+    curstat = {
+      :name => name,
+      :timestamp => ts.to_i,
+      :hash => hash,
+      :changes => []
+    }
+  else
+    adds, deletes, file = line.split(/\s+/)
+    curstat[:changes] << [
+      adds.to_i, deletes.to_i
+    ]
+  end
+end

data/bin/instrument_server

@@ -0,0 +1,267 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+gem 'instrumental_agent'
+require 'instrumental_agent'
+
+
+class SystemInspector
+  TYPES = [:gauges, :incrementors]
+  attr_accessor *TYPES
+
+  def initialize
+    @gauges = {}
+    @incrementors = {}
+    @platform =
+      case RUBY_PLATFORM
+      when /linux/
+        Linux
+      when /darwin/
+        OSX
+      else
+        raise "unsupported OS"
+      end
+  end
+
+  def load_all
+    load @platform.load_cpu
+    load @platform.load_memory
+    load @platform.load_disks
+    load @platform.load_filesystem
+  end
+
+  def load(stats)
+    @gauges.merge!(stats[:gauges] || {})
+  end
+
+  module OSX
+    def self.load_cpu
+      { :gauges => top }
+    end
+
+    def self.top
+      lines = []
+      processes = date = load = cpu = nil
+      IO.popen('top -l 1 -n 0') do |top|
+        processes = top.gets.split(': ')[1]
+        date = top.gets
+        load = top.gets.split(': ')[1]
+        cpu = top.gets.split(': ')[1]
+      end
+
+      user, system, idle = cpu.split(", ").map { |v| v.to_f }
+      load1, load5, load15 = load.split(", ").map { |v| v.to_f }
+      total, running, stuck, sleeping, threads = processes.split(", ").map { |v| v.to_i }
+
+      {
+        'cpu.user' => user,
+        'cpu.system' => system,
+        'cpu.idle' => idle,
+        'load.1min' => load1,
+        'load.5min' => load5,
+        'load.15min' => load15,
+        'processes.total' => total,
+        'processes.running' => running,
+        'processes.stuck' => stuck,
+        'processes.sleeping' => sleeping,
+        'threads' => threads,
+      }
+    end
+
+    def self.load_memory
+      # TODO: swap
+      { :gauges => vm_stat }
+    end
+
+    def self.vm_stat
+      header, *rows = `vm_stat`.split("\n")
+      page_size = header.match(/page size of (\d+) bytes/)[1].to_i
+      sections = ["free", "active", "inactive", "wired", "speculative", "wired down"]
+      output = {}
+      total = 0.0
+      rows.each do |row|
+        if match = row.match(/Pages (.*):\s+(\d+)\./)
+          section, value = match[1, 2]
+          if sections.include?(section)
+            value = value.to_f * page_size / 1024 / 1024
+            output["memory.#{section.gsub(' ', '_')}_mb"] = value
+            total += value
+          end
+        end
+      end
+      output["memory.free_percent"] = output["memory.free_mb"] / total * 100 # TODO: verify
+      output
+    end
+
+    def self.load_disks
+      { :gauges => df }
+    end
+
+    def self.df
+      output = {}
+      `df -k`.split("\n").grep(%r{^/dev/}).each do |line|
+        device, total, used, available, capacity, mount = line.split(/\s+/)
+        names = [File.basename(device)]
+        names << 'root' if mount == '/'
+        names.each do |name|
+          output["disk.#{name}.total_mb"] = total.to_f / 1024
+          output["disk.#{name}.used_mb"] = used.to_f / 1024
+          output["disk.#{name}.available_mb"] = available.to_f / 1024
+          output["disk.#{name}.available_percent"] = available.to_f / total.to_f * 100
+        end
+      end
+      output
+    end
+
+    def self.netstat(interface = 'en1')
+      # mostly functional network io stats
+      headers, *lines = `netstat -ibI #{interface}`.split("\n").map { |l| l.split(/\s+/) } # FIXME: vulnerability?
+      headers = headers.map { |h| h.downcase }
+      lines.each do |line|
+        if !line[3].include?(':')
+          return Hash[headers.zip(line)]
+        end
+      end
+    end
+
+    def self.load_filesystem
+      {}
+    end
+
+  end
+
+  module Linux
+    def self.load_cpu
+      output = { :gauges => {} }
+      output[:gauges].merge!(cpu)
+      output[:gauges].merge!(loadavg)
+      output
+    end
+
+    def self.cpu
+      cpu, user, nice, system, idle, iowait = `cat /proc/stat | grep cpu[^0-9]`.chomp.split(/\s+/)
+      total = user.to_i + system.to_i + idle.to_i + iowait.to_i
+      {
+        'cpu.user' => (user.to_f / total) * 100,
+        'cpu.system' => (system.to_f / total) * 100,
+        'cpu.idle' => (idle.to_f / total) * 100,
+        'cpu.iowait' => (iowait.to_f / total) * 100,
+      }
+    end
+
+    def self.loadavg
+      min_1, min_5, min_15 = `cat /proc/loadavg`.split(/\s+/)
+      {
+        'load.1min' => min_1.to_f,
+        'load.5min' => min_5.to_f,
+        'load.15min' => min_15.to_f,
+      }
+    end
+
+    def self.load_memory
+      output = { :gauges => {} }
+      output[:gauges].merge!(memory)
+      output[:gauges].merge!(swap)
+      output
+    end
+
+    def self.memory
+      _, total, used, free, shared, buffers, cached = `free -k -o | grep Mem`.chomp.split(/\s+/)
+      {
+        'memory.used_mb' => used.to_f / 1024,
+        'memory.free_mb' => free.to_f / 1024,
+        'memory.buffers_mb' => buffers.to_f / 1024,
+        'memory.cached_mb' => cached.to_f / 1024,
+        'memory.free_percent' => (free.to_f / total.to_f) * 100,
+      }
+    end
+
+    def self.swap
+      _, total, used, free = `free -k -o | grep Swap`.chomp.split(/\s+/)
+      {
+        'swap.used_mb' => used.to_f / 1024,
+        'swap.free_mb' => free.to_f / 1024,
+        'swap.free_percent' => (free.to_f / total.to_f) * 100,
+      }
+    end
+
+    def self.load_disks
+      { :gauges => disks.merge(iostat) }
+    end
+
+    def self.disks
+      output = {}
+      `df -Pk`.split("\n").grep(%r{^/dev/}).each do |line|
+        device, total, used, available, capacity, mount = line.split(/\s+/)
+        names = [File.basename(device)]
+        names << 'root' if mount == '/'
+        names.each do |name|
+          output["disk.#{name}.total_mb"] = total.to_f / 1024
+          output["disk.#{name}.used_mb"] = used.to_f / 1024
+          output["disk.#{name}.available_mb"] = available.to_f / 1024
+          output["disk.#{name}.available_percent"] = available.to_f / total.to_f * 100
+        end
+      end
+      output
+    end
+
+    def self.iostat
+      output = {}
+      if system('iostat > /dev/null 2>&1')
+        lines      = `iostat -dx`.split("\n")
+        header     = lines.grep(/^Device/i).first
+        dev_lines  = lines[(lines.index(header) + 1)..-1]
+        util_index = header.split.index('%util')
+        dev_lines.each do |dev_line|
+          values = dev_line.split
+          output["disk.#{values[0]}.percent_utilization"] = values[util_index].to_f
+        end
+      end
+      output
+    end
+
+    def self.load_filesystem
+      { :gauges => filesystem }
+    end
+
+    def self.filesystem
+      allocated, unused, max = `sysctl fs.file-nr`.split[-3..-1].map(&:to_i)
+      open_files = allocated - unused
+      { 'filesystem.open_files' => open_files,
+        'filesystem.max_open_files' => max,
+        'filesystem.open_files_pct_max' => (open_files.to_f / max.to_f) * 100,
+      }
+    end
+  end
+
+end
+
+token = ENV["INSTRUMENTAL_TOKEN"]
+if !token
+  token, collector = *ARGV
+else
+  collector = *ARGV
+end
+unless token
+  puts "Usage: #{$0} <token> [collector]"
+  exit 1
+end
+I = Instrumental::Agent.new(token, :collector => collector)
+
+host = `hostname`.chomp
+
+puts "Collecting stats under the hostname: #{host}"
+
+unless system('iostat > /dev/null 2>&1')
+  puts 'Install iostat (sysstat package) to collect disk I/O metrics.'
+end
+
+loop do
+  inspector = SystemInspector.new
+  inspector.load_all
+  inspector.gauges.each do |stat, value|
+    I.gauge("#{host}.#{stat}", value)
+  end
+  # I.increment("#{host}.#{stat}", delta)
+  sleep 10
+end

data/bin/instrumental

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+require 'cgi'
+require 'net/http'
+require 'optparse'
+require 'rubygems'
+gem 'json'
+require 'json'
+
+
+metrics = "*"
+token = ENV["INSTRUMENTAL_TOKEN"]
+allowed_durations = [1800, 3600, 10800, 21600, 43200, 86400, 259200, 604800, 1209600]
+duration = nil
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: instrumental [options]"
+
+  opts.on("-m", "--metrics METRICS", "Comma separated list of metrics you're interested in.  Wildcard (*) allowed") do |m|
+    metrics = m
+  end
+
+  opts.on("-t", "--token TOKEN", "Your API token.  Also can be set as an environment variable, INSTRUMENTAL_TOKEN") do |t|
+    token = t
+  end
+
+  opts.on("-d", "--duration DURATION", "Duration ago in seconds to view.  You can specify #{allowed_durations.join(" ")}") do |d|
+    duration = allowed_durations.index(d.to_i) && d.to_i
+  end
+end.parse!
+
+search_line = CGI.escape(metrics)
+duration &&= "&duration=#{duration}"
+http = Net::HTTP.new("instrumentalapp.com", 80)
+request = Net::HTTP::Get.new("/metrics/#{search_line}?token=#{token}&format=json#{duration}")
+response = http.request(request)
+raise "Server error: #{response.message} #{response.code}" if response.code.to_i != 200
+response = JSON.parse(response.body)
+bars = ["\342\226\201", "\342\226\202", "\342\226\203", "\342\226\204", "\342\226\205", "\342\226\206", "\342\226\207"]
+max_key_size = response.collect { |k, v| k.size }.max
+scale = bars.size - 1
+response.each do |metric, data|
+  data = data.collect { |h| h["value"].to_f }
+  min = data.min
+  range = data.max - min
+  graph = data.collect do |v|
+    if (idx = (((v - min) / range) * scale)).nan?
+      idx = 0
+    elsif idx.infinite?
+      idx = scale
+    end
+    bars[idx.round]
+  end
+  puts "%#{max_key_size}s | %s" % [metric, graph.join]
+end

data/instrumental_tools.gemspec

@@ -0,0 +1,20 @@
+$: << "./lib"
+require 'instrumental_tools/version'
+
+Gem::Specification.new do |s|
+  s.name        = "instrumental_tools"
+  s.version     = Instrumental::Tools::VERSION
+  s.authors     = ["Elijah Miller", "Christopher Zelenak", "Kristopher Chambers", "Matthew Hassfurder"]
+  s.email       = ["support@instrumentalapp.com"]
+  s.homepage    = "http://github.com/fastestforward/instrumental_tools"
+  s.summary     = %q{Command line tools for Instrumental}
+  s.description = %q{Tools for displaying information from and reporting to Instrumental (instrumentalapp.com)}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_runtime_dependency(%q<json>, [">=0"])
+  s.add_runtime_dependency(%q<instrumental_agent>, [">=0.5"])
+  s.add_development_dependency(%q<rake>, [">=0"])
+end

data/lib/instrumental_tools/version.rb

@@ -0,0 +1,5 @@
+module Instrumental
+  module Tools
+    VERSION = "0.1.0"
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: instrumental_tools
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Elijah Miller
+- Christopher Zelenak
+- Kristopher Chambers
+- Matthew Hassfurder
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &70344909872300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70344909872300
+- !ruby/object:Gem::Dependency
+  name: instrumental_agent
+  requirement: &70344909871780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70344909871780
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70344909871300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70344909871300
+description: Tools for displaying information from and reporting to Instrumental (instrumentalapp.com)
+email:
+- support@instrumentalapp.com
+executables:
+- gitstrumental
+- instrument_server
+- instrumental
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/gitstrumental
+- bin/instrument_server
+- bin/instrumental
+- instrumental_tools.gemspec
+- lib/instrumental_tools/version.rb
+homepage: http://github.com/fastestforward/instrumental_tools
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 266192678575170194
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 266192678575170194
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Command line tools for Instrumental
+test_files: []