bipbip 0.0.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ffb2e47b0d064209e1459b7be4036cc1ab15f2bb
+  data.tar.gz: 6da35aa18a085d1bfcc572c5d7197d903b7a4ee1
+SHA512:
+  metadata.gz: 3ef24076ec298fc436416646b4c09d936b767282d93fc77e083da7345d7ce506efd8cb3915b93997390038c5030169ec66e5f5df80841ef32aac5df397882478
+  data.tar.gz: 3c6330c8064fe34ca717e177e7f6e1be8f86b0008cd02ace5f3e46a286381a6e0e80f7c2d2690faac28fba85fadfe296cb5e1c5c0a495802ba62de387cfb0eae

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Cargo Media
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,55 @@
+bipbip
+======
+Agent to collect server metrics and send them to the [CopperEgg RevealMetrics](http://copperegg.com/) platform.
+Plugins for different metrics available in the `plugin/`-directory.
+Will spawn a child process for every plugin and server you tell it to monitor.
+
+Configure and run
+-----------------
+Pass the path to your configuration file to `bipbip` using the `-c` command line argument.
+```sh
+bipbip -c /etc/bipbip/bipbip.yml
+```
+
+The configuration file should list the services you want to collect for, and the servers for each of them, e.g.:
+```yml
+loglevel: INFO
+copperegg:
+  apikey: YOUR_APIKEY
+  frequency: 15
+
+services:
+  -
+    plugin: Memcached
+    hostname: localhost
+    port: 11211
+  -
+    plugin: Mysql
+    hostname: localhost
+    port: 3306
+    username: root
+    password: root
+  -
+    plugin: Redis
+    hostname: localhost
+    port: 6379
+  -
+    plugin: Gearman
+    hostname: localhost
+    port: 4730
+```
+
+Include configuration
+---------------------
+In your configuration you can specify a directory to include service configurations from:
+```
+include: services.d/
+```
+This will include files from `/etc/bipbip/services.d/` and load them into the `services` configuration.
+
+You could then add a file `/etc/bipbip/services.d/memcached.yml`:
+```yml
+plugin: Memcached
+hostname: localhost
+port: 11211
+```

data/bin/bipbip

@@ -0,0 +1,42 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../lib')
+require 'bipbip'
+require 'optparse'
+
+name = 'bipbip'
+version = Bipbip::VERSION
+
+options = {}
+optparse = OptionParser.new do |opts|
+  opts.banner = "Usage: #{name} [options]"
+
+  opts.on("-c", "--config PATH", "Configuration file") do |c|
+    options[:config] = c
+  end
+
+  opts.on("-v", "--version", "Version") do |v|
+    puts "#{name} v#{version}"
+    exit
+  end
+
+  opts.on("-h", "--help", "Show this help") do |h|
+    puts opts
+    exit
+  end
+end
+
+
+begin
+  optparse.parse!
+  if not options[:config]
+    puts "Missing options: config file [-c PATH]"
+    exit
+  end
+rescue OptionParser::InvalidOption, OptionParser::MissingArgument
+  puts $!.to_s
+  puts optparse
+  exit
+end
+
+Bipbip::Agent.new.run options[:config]

data/lib/bipbip.rb

@@ -0,0 +1,29 @@
+module Bipbip
+  require 'rubygems'  # For ruby < 1.9
+
+  require 'copperegg'
+  require 'yaml'
+  require 'logger'
+  require 'socket'
+
+  require 'bipbip/version'
+  require 'bipbip/interruptible_sleep'
+  require 'bipbip/agent'
+  require 'bipbip/plugin'
+  require 'bipbip/plugin/memcached'
+  require 'bipbip/plugin/mysql'
+  require 'bipbip/plugin/redis'
+  require 'bipbip/plugin/gearman'
+
+  def self.logger
+    @logger
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  def self.fqdn
+    @fqdn ||= Socket.gethostbyname(Socket.gethostname).first
+  end
+end

data/lib/bipbip/agent.rb

@@ -0,0 +1,103 @@
+module Bipbip
+
+  class Agent
+
+    def initialize
+      @plugin_pids = []
+    end
+
+    def run(config_file)
+      config = YAML.load(File.open(config_file))
+
+      Bipbip.logger = Logger.new(STDOUT)
+      Bipbip.logger.level = Logger::const_get(config['loglevel'] || 'INFO')
+      Bipbip.logger.info 'Startup...'
+
+      CopperEgg::Api.apikey = config['copperegg']['apikey']
+      CopperEgg::Api.host = config['copperegg']['host'] if !config['copperegg']['host'].nil?
+      frequency = config['copperegg']['frequency'].to_i
+
+      if ![5, 15, 60, 300, 900, 3600, 21600].include?(frequency)
+        Bipbip.logger.fatal "Invalid frequency: #{frequency}"
+        exit
+      end
+
+      ['INT', 'TERM'].each { |sig| trap(sig) {
+        Thread.new { interrupt }
+      } }
+
+      services = config['services']
+      if config.has_key?('include')
+        include_path = File.expand_path(config['include'], File.dirname(config_file))
+        services += load_include_configs(include_path)
+      end
+
+      metric_groups = load_metric_groups
+      dashboards = load_dashboards
+
+      plugin_names = services.map { |service| service['plugin'] }
+      plugin_names.each do |plugin_name|
+        plugin = Plugin::const_get(plugin_name).new
+
+        metric_group = metric_groups.detect { |m| m.name == plugin_name }
+        if metric_group.nil? || !metric_group.is_a?(CopperEgg::MetricGroup)
+          Bipbip.logger.info "Creating metric group `#{plugin_name}`"
+          metric_group = CopperEgg::MetricGroup.new(:name => plugin_name, :label => plugin_name, :frequency => frequency)
+        end
+        metric_group.frequency = frequency
+        metric_group.metrics = plugin.metrics_schema
+        metric_group.save
+
+        dashboard = dashboards.detect { |d| d.name == plugin_name }
+        if dashboard.nil?
+          Bipbip.logger.info "Creating dashboard `#{plugin_name}`"
+          metrics = metric_group.metrics || []
+          CopperEgg::CustomDashboard.create(metric_group, :name => plugin_name, :identifiers => nil, :metrics => metrics)
+        end
+      end
+
+      services.each do |service|
+        plugin_name = service['plugin']
+        Bipbip.logger.info "Starting plugin #{plugin_name}"
+        plugin = Plugin::const_get(plugin_name).new
+        @plugin_pids.push plugin.run(service, frequency)
+      end
+
+      p Process.waitall
+    end
+
+    def load_metric_groups
+      Bipbip.logger.info 'Loading metric groups'
+      metric_groups = CopperEgg::MetricGroup.find
+      if metric_groups.nil?
+        raise 'Cannot load metric groups'
+      end
+      metric_groups
+    end
+
+    def load_dashboards
+      Bipbip.logger.info 'Loading dashboards'
+      dashboards = CopperEgg::CustomDashboard.find
+      if dashboards.nil?
+        raise 'Cannot load dashboards'
+      end
+      dashboards
+    end
+
+    def load_include_configs(directory)
+      files = Dir[directory + '/**/*.yaml', directory + '/**/*.yml']
+      services = files.map {|file| YAML.load(File.open(file))}
+    end
+
+    def interrupt
+      Bipbip.logger.info 'Interrupt, killing plugin processes...'
+      @plugin_pids.each { |pid| Process.kill('TERM', pid) }
+
+      Bipbip.logger.info 'Waiting for all plugin processes to exit...'
+      Process.waitall
+
+      Bipbip.logger.info 'Exiting'
+      exit
+    end
+  end
+end

data/lib/bipbip/interruptible_sleep.rb

@@ -0,0 +1,10 @@
+module InterruptibleSleep
+  def interruptible_sleep(seconds)
+    @_sleep_check, @_sleep_interrupt = IO.pipe
+    IO.select([@_sleep_check], nil, nil, seconds)
+  end
+
+  def interrupt_sleep
+    @_sleep_interrupt.close if @_sleep_interrupt and !@_sleep_interrupt.closed?
+  end
+end

data/lib/bipbip/plugin.rb

@@ -0,0 +1,65 @@
+module Bipbip
+
+  class Plugin
+    include InterruptibleSleep
+
+    def initialize
+    end
+
+    def run(server, frequency)
+      child_pid = fork do
+        ['INT', 'TERM'].each { |sig| trap(sig) {
+          Thread.new { interrupt } if !@interrupted
+        } }
+
+        retry_delay = frequency
+        metric_identifier = metric_identifier(server)
+        begin
+          until interrupted? do
+            time = Time.now
+            data = monitor(server)
+            Bipbip.logger.debug "#{name} #{metric_identifier}: Data: #{data}"
+            CopperEgg::MetricSample.save(name, metric_identifier, Time.now.to_i, data)
+            retry_delay = frequency
+            interruptible_sleep (frequency - (Time.now - time))
+          end
+        rescue => e
+          Bipbip.logger.error "#{name} #{metric_identifier}: Error getting data: #{e.inspect}"
+          interruptible_sleep retry_delay
+          retry_delay += frequency if retry_delay < frequency * 10
+          retry
+        end
+      end
+    end
+
+    def interrupt
+      Bipbip.logger.info "Interrupting plugin process #{Process.pid}"
+      @interrupted = true
+      interrupt_sleep
+    end
+
+    def interrupted?
+      @interrupted || Process.getpgid(Process.ppid) != Process.getpgrp
+    end
+
+    def name
+      self.class.name.split('::').last
+    end
+
+    def metric_identifier(server)
+      Bipbip.fqdn + '::' + server['hostname']
+    end
+
+    def metrics_names
+      metrics_schema.map {|metric| metric[:name] }
+    end
+
+    def metrics_schema
+      raise 'Missing method metrics_schema'
+    end
+
+    def monitor(server)
+      raise 'Missing method monitor'
+    end
+  end
+end

data/lib/bipbip/plugin/gearman.rb

@@ -0,0 +1,29 @@
+require 'gearman/server'
+class GearmanServer < Gearman::Server
+end
+
+module Bipbip
+
+  class Plugin::Gearman < Plugin
+
+    def metrics_schema
+      [
+          {:name => 'jobs_queued_total', :type => 'ce_gauge', :unit => 'Jobs'},
+      ]
+    end
+
+    def monitor(server)
+      gearman = GearmanServer.new(server['hostname'] + ':' + server['port'].to_s)
+      stats = gearman.status
+
+      jobs_queued_total = 0
+      stats.each do |function_name, data|
+        data.each do |queue, stats|
+          jobs_queued_total += queue.to_i
+        end
+      end
+
+      {:jobs_queued_total => jobs_queued_total}
+    end
+  end
+end

data/lib/bipbip/plugin/memcached.rb

@@ -0,0 +1,30 @@
+require 'memcached'
+class MemcachedClient < Memcached
+end
+
+module Bipbip
+
+  class Plugin::Memcached < Plugin
+
+    def metrics_schema
+      [
+          {:name => 'cmd_get', :type => 'ce_counter'},
+          {:name => 'cmd_set', :type => 'ce_counter'},
+          {:name => 'get_misses', :type => 'ce_counter'},
+          {:name => 'limit_maxbytes', :type => 'ce_gauge', :unit => 'b'},
+          {:name => 'bytes', :type => 'ce_gauge', :unit => 'b'},
+      ]
+    end
+
+    def monitor(server)
+      cache = MemcachedClient.new(server['hostname'] + ':' + server['port'].to_s)
+      stats = cache.stats
+
+      data = {}
+      metrics_names.each do |key|
+        data[key] = stats[key.to_sym].shift.to_i
+      end
+      data
+    end
+  end
+end

data/lib/bipbip/plugin/mysql.rb

@@ -0,0 +1,75 @@
+require 'mysql2'
+
+module Bipbip
+
+  class Plugin::Mysql < Plugin
+
+    def metrics_schema
+      [
+          {:name => 'max_connections', :type => 'ce_gauge', :unit => 'Connections'},
+          {:name => 'Max_used_connections', :type => 'ce_gauge', :unit => 'Connections'},
+          {:name => 'Connections', :type => 'ce_counter', :unit => 'Connections'},
+          {:name => 'Threads_connected', :type => 'ce_gauge', :unit => 'Threads'},
+
+          {:name => 'Seconds_Behind_Master', :type => 'ce_gauge', :unit => 'Seconds'},
+
+          {:name => 'Created_tmp_disk_tables', :type => 'ce_counter', :unit => 'Tables'},
+
+          {:name => 'Queries', :type => 'ce_counter', :unit => 'Queries'},
+          {:name => 'Slow_queries', :type => 'ce_counter', :unit => 'Queries'},
+
+          {:name => 'Bytes_received', :type => 'ce_counter', :unit => 'b'},
+          {:name => 'Bytes_sent', :type => 'ce_counter', :unit => 'b'},
+
+          {:name => 'Table_locks_immediate', :type => 'ce_counter', :unit => 'Locks'},
+          {:name => 'Table_locks_waited', :type => 'ce_counter', :unit => 'Locks'},
+
+          {:name => 'Processlist', :type => 'ce_gauge', :unit => 'Processes'},
+          {:name => 'Processlist_Locked', :type => 'ce_gauge', :unit => 'Processes'},
+          {:name => 'Processlist_Sending_data', :type => 'ce_gauge', :unit => 'Processes'},
+
+          {:name => 'Com_select', :type => 'ce_counter', :unit => 'Commands'},
+          {:name => 'Com_delete', :type => 'ce_counter', :unit => 'Commands'},
+          {:name => 'Com_insert', :type => 'ce_counter', :unit => 'Commands'},
+          {:name => 'Com_update', :type => 'ce_counter', :unit => 'Commands'},
+          {:name => 'Com_replace', :type => 'ce_counter', :unit => 'Commands'},
+      ]
+    end
+
+    def monitor(server)
+      mysql = Mysql2::Client.new(
+          :host => server['hostname'],
+          :port => server['port'],
+          :username => server['username'],
+          :password => server['password']
+      )
+
+      stats = Hash.new(0)
+
+      mysql.query('SHOW GLOBAL STATUS').each do |v|
+        stats[v['Variable_name']] = v['Value'].to_i
+      end
+
+      mysql.query('SHOW VARIABLES').each do |v|
+        stats[v['Variable_name']] = v['Value'].to_i
+      end
+
+      slave_status = mysql.query('SHOW SLAVE STATUS').first
+      if slave_status
+        stats['Seconds_Behind_Master'] = slave_status['Seconds_Behind_Master'].to_i
+      end
+
+      processlist =  mysql.query('SHOW PROCESSLIST')
+      stats['Processlist'] = processlist.count
+      processlist.each do |process|
+        stats['Processlist_' + process['State'].sub(' ', '_')] += 1 unless process['State'].empty?
+      end
+
+      data = {}
+      metrics_names.each do |key|
+        data[key] = stats[key]
+      end
+      data
+    end
+  end
+end

data/lib/bipbip/plugin/redis.rb

@@ -0,0 +1,30 @@
+require 'redis'
+class RedisClient < Redis
+end
+
+module Bipbip
+
+  class Plugin::Redis < Plugin
+
+    def metrics_schema
+      [
+          {:name => 'total_commands_processed', :type => 'ce_counter', :unit => 'Commands'},
+          {:name => 'used_memory', :type => 'ce_gauge', :unit => 'b'},
+      ]
+    end
+
+    def monitor(server)
+      redis = RedisClient.new(
+          :host => server['hostname'],
+          :port => server['port']
+      )
+      stats = redis.info
+
+      data = {}
+      metrics_names.each do |key|
+        data[key] = stats[key].to_i
+      end
+      data
+    end
+  end
+end

data/lib/bipbip/version.rb

@@ -0,0 +1,3 @@
+module Bipbip
+  VERSION = '0.0.7'
+end

metadata

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: bipbip
+version: !ruby/object:Gem::Version
+  version: 0.0.7
+platform: ruby
+authors:
+- Cargo Media
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: copperegg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: memcached
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gearman-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Agent to collect data for common server programs and push them to CopperEgg
+email: hello@cargomedia.ch
+executables:
+- bipbip
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/bipbip
+- lib/bipbip/agent.rb
+- lib/bipbip/interruptible_sleep.rb
+- lib/bipbip/plugin/gearman.rb
+- lib/bipbip/plugin/memcached.rb
+- lib/bipbip/plugin/mysql.rb
+- lib/bipbip/plugin/redis.rb
+- lib/bipbip/plugin.rb
+- lib/bipbip/version.rb
+- lib/bipbip.rb
+homepage: https://github.com/cargomedia/bipbip
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Gather services data and store in CopperEgg
+test_files: []