fiveruns-dash-sensor 0.8.2

data/History.rdoc

@@ -0,0 +1,5 @@
+= Changelog
+
+== 0.8.0 (2009-01-07)
+
+* Initial internal release.

data/README.rdoc

@@ -0,0 +1,71 @@
+= fiveruns-dash-sensor
+
+Please see http://dash.fiveruns.com for an overview of the Dash service.
+
+== Summary
+
+Sensor provides a mechanism for "external" monitoring of infrastructure with Dash.
+Since Dash is usually embedded within an application (i.e. internal), there's no easy
+way to monitor things like Apache, memcached, or other pieces.  Sensor is designed to
+be installed on a machine and upload metrics for any infrastructure on that machine.
+The user just needs to configure which elements they want to monitor.
+
+== Installation
+
+gem install fiveruns-dash-sensor -s http://gems.github.com
+
+== Configuration
+
+When you run 'fiveruns-dash-sensor start' for the first time, it will create a blank
+configuration for you in ~/.fiveruns-dash-sensor/config.rb.  You will need to edit
+this file with your app token and activate the plugins for your environment.  Once
+done, you should be able to run 'fiveruns-dash-sensor run' for a few minutes to verify 
+everything is working as designed.  If there are no problems, you can run
+'fiveruns-dash-sensor start' to spin off the daemon into the background and forget about it.
+
+== Command Summary
+
+'fiveruns-dash-sensor run' - start the daemon in the foreground, for debugging purposes.
+'fiveruns-dash-sensor start' - start the daemon in the background, output and PID file goes
+ in the ~/.fiveruns-dash-sensor/log/ directory.
+'fiveruns-dash-sensor stop' - stop the daemon in the background
+
+You can run Sensor in verbose mode to get more debug output:
+
+'fiveruns-dash-sensor run -- -v'
+
+== Creating your own Plugins
+
+Sensor allows you to create your own custom plugins for your own infrastructure.  Each
+plugin should reside in ~/.fiveruns-dash-sensor/<name>.rb and config.rb should have an
+entry in it like this:
+
+sensor.plugin :<name>, any options here...
+
+Please see the existing plugins for examples of how your code should work.
+
+
+== License
+
+# (The FiveRuns License)
+#
+# Copyright (c) 2009 FiveRuns Corporation
+# 
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/fiveruns-dash-sensor

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+
+unless File.exist?("#{ENV['HOME']}/.fiveruns-dash-sensor/config.rb")
+	local = File.join(File.dirname(__FILE__), "..", "config.rb")
+	dir = File.expand_path("~/.fiveruns-dash-sensor")
+	if File.exist?(local)
+		require 'fileutils'
+		FileUtils.mkdir_p dir
+		FileUtils.cp local, dir
+		puts ""
+		puts "Configuration Required"
+		puts ""
+		puts "A blank configuration file has been placed in ~/.fiveruns-dash-sensor/config.rb"
+		puts "Please edit this file to add your application token and configure the plugins"
+		puts "appropriate to this environment."
+		puts ""
+	end
+	exit
+end
+
+root = File.expand_path("~/.fiveruns-dash-sensor")
+log_dir = File.join(root, "log")
+unless File.exist? log_dir
+	require 'fileutils'
+	FileUtils.mkdir_p log_dir 
+end
+
+gem 'daemons'
+require 'daemons'
+
+options = {
+	:app_name   => "dash-sensor",
+	:dir_mode   => :normal,
+	:dir        => log_dir,
+	:multiple   => true,
+	:backtrace  => true,
+	:monitor    => false,
+	:log_output => true
+}
+
+Daemons.run(File.join(File.dirname(__FILE__), '..', 'lib', 'daemon.rb'), options)

data/config.rb

@@ -0,0 +1,19 @@
+# Example configuration for FiveRuns Dash Sensor daemon.
+# Configure your Dash sensor instance here.
+# This file should reside in ~/.fiveruns-dash-sensor/config.rb
+#
+# We suggest you have a single Dash Sensor instance running per machine
+# and an application token per environment.  So if you have 8 machines
+# in your application's production cluster, you would have 8 sensor instances
+# running for one application named "<App> Production Cluster".
+
+#sensor.token = 'change-to-your-application-token'
+
+# One line per piece of infrastructure you wish to monitor.
+# The plugins are in the plugins directory.
+
+# Available options and their defaults are listed.
+#sensor.plugin 'memcached', :iface => 'localhost', :port => 11211
+#sensor.plugin 'starling', :iface => 'localhost', :port => 22122
+#sensor.plugin 'apache', :url => 'http://localhost/server-status?auto'
+#sensor.plugin 'nginx', :url => 'http://localhost/nginx_status'

data/lib/daemon.rb

@@ -0,0 +1,46 @@
+require 'ostruct'
+require 'optparse'
+
+# Sent by daemons when you run '<script> stop'
+Signal.trap('TERM') { puts "fiveruns-dash-sensor PID #{$$} exiting at #{Time.now}..."; exit(0) }
+# Sent by daemons when you hit Ctrl-C after '<script> run'
+Signal.trap('INT') { puts "fiveruns-dash-sensor terminated at #{Time.now}..."; exit(0) }
+
+options = OpenStruct.new
+options.environment = ENV['RAILS_ENV'] || 'production'
+options.verbose = false
+options.config_file = "#{ENV['HOME']}/.fiveruns-dash-sensor/config.rb"
+
+op = OptionParser.new do |opts|
+  opts.banner = "Usage: fiveruns-dash-sensor [options]"
+  opts.separator "General Options:"
+  opts.on("-e ENVIRONMENT", "--environment NAME", "Select environment [default: #{options.environment}]") do |v|
+    options.environment = v
+  end
+  opts.on("-c CONFIG", "--config FILE", "Select config file [default: #{options.config_file}]") do |v|
+    options.config_file = v
+  end
+  opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+    options.verbose = v
+  end
+  opts.separator "Other Options:"
+  opts.on("-h", "--help", "Display this message") do |v|
+    STDERR.puts opts
+    exit
+  end
+end
+op.parse!(ARGV)
+
+unless File.exist? options.config_file
+  puts "Please create a configuration file for your environment in ~/.fiveruns-dash-sensor/config.rb"
+  exit(1)
+end
+
+require 'logger'
+LOG = Logger.new(STDOUT)
+LOG.level = options.verbose ? Logger::DEBUG : Logger::INFO
+
+$LOAD_PATH.unshift File.dirname(__FILE__)
+require "sensor"
+LOG.info("Starting Dash Sensor [#{$$}] at #{Time.now}")
+Dash::Sensor::Engine.new.start(options)

data/lib/sensor.rb

@@ -0,0 +1,93 @@
+require 'fiveruns/dash'
+require 'sensor_plugin'
+
+module Dash
+  module Sensor
+
+    RECIPES = []
+
+    class Engine
+
+      def self.registered(name, url, klass)
+        klass.instance = klass.new
+        RECIPES << [name, url, klass]
+      end
+
+      def start(options)
+        load_plugins(options)
+        wait_forever
+      end
+
+      private
+
+      def load_plugins(options)
+        # Add the user's config directory to the load path so they can load custom plugins
+        # without modifying this gem.
+        $PLUGIN_PATH = [
+          File.dirname(options.config_file), 
+          File.expand_path(File.join(File.dirname(__FILE__), '..', 'plugins')),
+          nil
+        ]
+
+        setup = Setup.new
+        config = File.read(options.config_file)
+        setup.instance_eval(config, options.config_file)
+
+        start_dash(setup)
+      end
+
+      def start_dash(setup)
+        LOG.info("Configured Dash Sensor for #{setup.name} [#{setup.token}]")
+        Fiveruns::Dash.configure :app => setup.token do |config|
+          RECIPES.each do |(name, url, _)|
+            config.add_recipe name.to_sym, url
+          end
+        end
+        Fiveruns::Dash.session.reporter.interval = 60
+        Fiveruns::Dash.session.start
+      end
+
+      def wait_forever
+        while true
+          sleep 60
+        end
+      end
+    end
+
+    class Setup
+      
+      attr_accessor :token, :name
+
+      def initialize
+        self.name = 'My App Name'
+        self.token = 'change-me-to-your-app-token'
+      end
+      
+      def plugin(name, options={})
+        LOG.info("Loading plugin #{name} with options #{options.inspect}")
+        
+        $PLUGIN_PATH.each do |path|
+          raise ArgumentError, "Unable to find #{name}.rb in plugin path: #{$PLUGIN_PATH[0..-2].inspect}" unless path
+          file = File.join(path, name)
+          if File.exist?("#{file}.rb")
+            LOG.debug "Loading #{file}"
+            require file
+            break
+          end
+        end
+
+        RECIPES.last.last.instance.configure(options)
+      end
+
+      def url=(value)
+        ENV['DASH_UPDATE'] = value
+      end
+
+      def sensor
+        self
+      end
+
+    end
+
+  end
+end

data/lib/sensor_plugin.rb

@@ -0,0 +1,16 @@
+module SensorPlugin
+  
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+  
+  module ClassMethods
+    attr_accessor :instance
+
+    def register(name, url, &block)
+      Fiveruns::Dash.register_recipe(name, url, &block)
+      Dash::Sensor::Engine.registered(name, url, self)
+    end
+  end
+  
+end

data/plugins/apache.rb

@@ -0,0 +1,56 @@
+require 'open-uri'
+
+module Dash::Sensor::Plugins
+  class Apache
+    include SensorPlugin
+    
+    ONE_KB = 1024
+
+    register :apache, :url => 'http://dash.fiveruns.com' do |recipe|
+      recipe.counter :requests, 'Requests' do
+        total = Integer(stats['Total Accesses'])
+        last = Integer(old_stats['Total Accesses'])
+        this_minute = total - last
+        this_minute > 0 && last > 0 ? this_minute : 0
+      end
+      recipe.counter :mbytes, 'Transferred', :unit => 'MB' do
+        total = Integer(stats['Total kBytes'])
+        last = Integer(old_stats['Total kBytes'])
+        this_minute = Float(total - last) / ONE_KB
+        this_minute > 0 && last > 0 ? this_minute : 0
+      end
+    end
+    
+    def configure(options)
+      @url = options.fetch(:url, 'http://localhost/server-status?auto')
+    end
+
+    private
+
+    def self.stats
+      if !@time || @time < Time.now - 55
+        @old_stats = @stats || Hash.new(0)
+        LOG.debug "Fetching status at #{Time.now}"
+        @stats = instance.send(:stats_data)
+        @time = Time.now
+      end
+      @stats
+    end
+    
+    def self.old_stats
+      @old_stats
+    end
+
+    def stats_data
+      begin
+        lines = open(@url).read.split("\n")
+        Hash[*lines.map {|line| line.split(':') }.flatten]
+      rescue => e
+        LOG.error "Error contacting #{@url}"
+        LOG.error "#{e.class.name}: #{e.message}"
+        Hash.new(0)
+      end
+    end
+
+  end
+end

data/plugins/memcached.rb

@@ -0,0 +1,65 @@
+module Dash::Sensor::Plugins
+  class Memcached
+    include SensorPlugin
+
+    register :memcached, :url => 'http://dash.fiveruns.com' do |recipe|
+      recipe.absolute :bytes, 'Cache Size', :unit => 'MB' do
+        Float(stats['bytes']) / (1024 * 1024)
+      end
+      recipe.percentage :hit_rate, 'Cache Hit Rate' do
+        hits = Integer(stats['get_hits'])
+        misses = Integer(stats['get_misses'])
+        total = hits + misses
+        total == 0 ? 0 : ((Float(hits) / total) * 100)
+      end
+    end
+
+    def configure(options)
+      @host = options.fetch(:iface, 'localhost')
+      @port = Integer(options.fetch(:port, 11211))
+    end
+
+    private
+    
+    def self.stats
+      if !@time || @time < Time.now - 55
+        LOG.debug "Fetching stats at #{Time.now}"
+        @stats = parse(instance.send(:stats_data))
+        @time = Time.now
+      end
+      @stats
+    end
+
+    def stats_data
+      data = ''
+      begin
+        sock = TCPSocket.new(@host, @port)
+        sock.print("stats\r\n")
+        sock.flush
+        # memcached does not close the socket once it is done writing
+        # the stats data.  We need to read line by line until we detect
+        # the END line and then stop/close on our side.
+        stats = sock.gets
+        while true
+          data += stats
+          break if stats.strip == 'END'
+          stats = sock.gets
+        end
+        sock.close
+      rescue Exception => e
+        LOG.error "Error contacting Starling at #{@host}:#{@port}"
+        LOG.error "#{e.class.name}: #{e.message}"
+      end
+      data
+    end
+
+    def self.parse(stats_data)
+      stats = Hash.new(0)
+      stats_data.each_line do |line|
+        stats[$1] = $2 if line =~ /STAT (\w+) (\S+)/
+      end
+      stats
+    end
+
+  end
+end

data/plugins/nginx.rb

@@ -0,0 +1,62 @@
+require 'open-uri'
+
+module Dash::Sensor::Plugins
+  class Nginx
+    include SensorPlugin
+    
+    ONE_KB = 1024
+
+    register :nginx, :url => 'http://dash.fiveruns.com' do |recipe|
+      recipe.counter :requests, 'Requests' do
+        total = stats[:requests]
+        last = old_stats[:requests]
+        this_minute = total - last
+        this_minute > 0 && last > 0 ? this_minute : 0
+      end
+      recipe.absolute :connections, 'Active Connections' do
+        stats[:active]
+      end
+    end
+    
+    def configure(options)
+      @url = options.fetch(:url, 'http://localhost/nginx-status')
+    end
+
+    private
+
+    def self.stats
+      if !@time || @time < Time.now - 55
+        @old_stats = @stats || Hash.new(0)
+        LOG.debug "Fetching status at #{Time.now}"
+        @stats = instance.send(:stats_data)
+        @time = Time.now
+      end
+      @stats
+    end
+    
+    def self.old_stats
+      @old_stats
+    end
+
+    def stats_data
+      begin
+        results = Hash.new(0)
+        data = open(@url).read
+        data.each_line do |line|
+          case line
+          when /^Active connections:\s+(\d+)/
+            results[:active] = Integer($1)
+          when /^\s+(\d+)\s+(\d+)\s+(\d+)/
+            results[:requests] = Integer($3)
+          end
+        end
+        results
+      rescue Exception => e
+        LOG.error "Error contacting #{@url}"
+        LOG.error "#{e.class.name}: #{e.message}"
+        Hash.new(0)
+      end
+    end
+
+  end
+end

data/plugins/starling.rb

@@ -0,0 +1,64 @@
+module Dash::Sensor::Plugins
+  class Starling
+    include SensorPlugin
+
+    register :starling, :url => 'http://dash.fiveruns.com' do |recipe|
+      recipe.absolute :processed, 'Processed Items' do
+        queues = stats.keys.map { |name| name =~ /queue_(\w+)_total_items/; $1 }.compact
+        queues.inject(0) { |total, queue_name| total + Integer(stats["queue_#{queue_name}_total_items"]) - Integer(stats["queue_#{queue_name}_items"]) }
+      end
+      recipe.absolute :queue_size, 'Current Size' do
+        queues = stats.keys.find_all { |name| name =~ /queue_\w+_items/ }
+        queues.inject(0) { |total, queue_name| total + Integer(stats[queue_name]) }
+      end
+    end
+    
+    def configure(options)
+      @host = options.fetch(:iface, 'localhost')
+      @port = Integer(options.fetch(:port, 22122))
+    end
+
+    private
+
+    def self.stats
+      if !@time || @time < Time.now - 55
+        LOG.debug "Fetching stats at #{Time.now}"
+        @stats = parse(instance.send(:stats_data))
+        @time = Time.now
+      end
+      @stats
+    end
+
+    def stats_data
+      data = ''
+      begin
+        sock = TCPSocket.new(@host, @port)
+        sock.print("stats\r\n")
+        sock.flush
+        # memcached does not close the socket once it is done writing
+        # the stats data.  We need to read line by line until we detect
+        # the END line and then stop/close on our side.
+        stats = sock.gets
+        while true
+          data += stats
+          break if stats.strip == 'END'
+          stats = sock.gets
+        end
+        sock.close
+      rescue Exception => e
+        LOG.error "Error contacting Starling at #{@host}:#{@port}"
+        LOG.error "#{e.class.name}: #{e.message}"
+      end
+      data
+    end
+
+    def self.parse(stats_data)
+      stats = Hash.new(0)
+      stats_data.each_line do |line|
+        stats[$1] = $2 if line =~ /STAT (\w+) (\S+)/
+      end
+      stats
+    end
+
+  end
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification 
+name: fiveruns-dash-sensor
+version: !ruby/object:Gem::Version 
+  version: 0.8.2
+platform: ruby
+authors: 
+- FiveRuns
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-02-05 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: Sensor allows Dash to monitor ad-hoc infrastructure non-invasively
+email: support@fiveruns.com
+executables: 
+- fiveruns-dash-sensor
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- History.rdoc
+files: 
+- bin/fiveruns-dash-sensor
+- config.rb
+- lib/daemon.rb
+- lib/sensor.rb
+- lib/sensor_plugin.rb
+- plugins/memcached.rb
+- plugins/starling.rb
+- plugins/nginx.rb
+- plugins/apache.rb
+- README.rdoc
+- History.rdoc
+has_rdoc: false
+homepage: http://github.com/fiveruns/dash-sensor/
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Sensor allows Dash to monitor ad-hoc infrastructure non-invasively
+test_files: []
+