rubix 0.0.1

data/lib/rubix/connection.rb

@@ -0,0 +1,111 @@
+require 'uri'
+require 'net/http'
+require 'json'
+
+require 'rubix/log'
+
+module Rubix
+
+  class Connection
+
+    include Logs
+
+    attr_reader :uri, :server, :auth, :request_id, :username, :password
+
+    def initialize uri_or_string, username=nil, password=nil
+      self.uri    = uri_or_string
+      @username   = username || uri.user
+      @password   = password || uri.password
+      @request_id = 0
+    end
+
+    def uri= uri_or_string
+      if uri_or_string.respond_to?(:host)
+        @uri = uri_or_string
+      else
+        string = uri_or_string =~ /^http/ ? uri_or_string : 'http://' + uri_or_string.to_s
+        @uri = URI.parse(string)
+      end
+      @server = Net::HTTP.new(uri.host, uri.port)
+    end
+
+    def authorized?
+      !auth.nil?
+    end
+
+    def authorization_params
+      {
+        :jsonrpc => "2.0",
+        :id      => request_id,
+        :method  => "user.login",
+        :params  => {
+          :user     => username,
+          :password => password
+        }
+      }
+    end
+
+    def authorize!
+      response = till_response { send_raw_request(authorization_params) }
+      raise AuthenticationError.new("Could not authenticate with Zabbix API at #{uri}: #{response.error_message}") if response.error?
+      raise AuthenticationError.new("Malformed response from Zabbix API: #{response.body}") unless response.string?
+      @auth        = response.result
+    end
+
+    def till_response attempt=1, max_attempts=5, &block
+      response = block.call
+      Rubix.logger.log(Logger::DEBUG, response.body, 'RECV') if Rubix.logger
+      case
+      when response.code.to_i >= 500 && attempt <= max_attempts
+        sleep 1                 # FIXME make the sleep time configurable...
+        till_response(attempt + 1, max_attempts, &block)
+      when response.code.to_i >= 500
+        raise ConnectionError.new("Too many consecutive failed requests (#{max_attempts}) to the Zabbix API at (#{uri}).")
+      else
+        Response.new(response)
+      end
+    end
+    
+    def request method, params
+      authorize! unless authorized?
+      till_response do
+        raw_params = {
+          :jsonrpc => "2.0",
+          :id      => request_id,
+          :method  => method,
+          :params  => params,
+          :auth    => auth
+        }
+        send_raw_request(raw_params)
+      end
+    end
+
+    def raw_post_request raw_params
+      json_body = raw_params.to_json
+      Rubix.logger.log(Logger::DEBUG, json_body, 'SEND') if Rubix.logger
+      Net::HTTP::Post.new(uri.path).tap do |req|
+        req['Content-Type'] = 'application/json-rpc'
+        req.body            = json_body
+      end
+    end
+
+    def host_with_port
+      if uri.port.nil? || uri.port.to_i == 80
+        uri.host
+      else
+        "#{uri.host}:#{uri.port}"
+      end
+    end
+
+    def send_raw_request raw_params
+      @request_id += 1
+      begin
+        raw_response = server.request(raw_post_request(raw_params))
+      rescue NoMethodError, SocketError => e
+        raise RequestError.new("Could not connect to Zabbix server at #{host_with_port}")
+      end
+      raw_response
+    end
+
+  end
+end

data/lib/rubix/examples/es_monitor.rb

@@ -0,0 +1,130 @@
+#!/usr/bin/env ruby
+
+RUBIX_ROOT = File.expand_path('../../../../lib', __FILE__)
+$: << RUBIX_ROOT unless $:.include?(RUBIX_ROOT)
+
+require 'rubix'
+require 'open-uri'
+
+class ESMonitor < Rubix::ClusterMonitor
+
+  # Hostgroup for any hosts that needs to be created.
+  HOSTGROUP = 'Elasticsearch clusters'
+
+  # Templates for any hosts that need to be created.
+  CLUSTER_TEMPLATES = 'Template_Elasticsearch_Cluster'
+  NODE_TEMPLATES    = 'Template_Elasticsearch_Node'
+
+  def node_query
+    'provides_service:*-elasticsearch'
+  end
+
+  def es_url private_ip, *args
+    "http://" + File.join(private_ip + ":9200", *args)
+  end
+  
+  def measure_cluster cluster_name
+    measured_cluster_health  = false
+    measured_cluster_indices = false
+    measured_cluster_nodes   = false
+    private_ips_by_cluster[cluster_name].each do |private_ip|
+      measured_cluster_health  = measure_cluster_health(cluster_name, private_ip)  unless measured_cluster_health
+      measured_cluster_indices = measure_cluster_indices(cluster_name, private_ip) unless measured_cluster_indices
+      measured_cluster_nodes   = measure_cluster_nodes(cluster_name, private_ip)   unless measured_cluster_nodes
+      break if measured_cluster_health && measured_cluster_indices && measured_cluster_nodes
+    end
+  end
+
+  # Measure the cluster health metrics -- /_cluster/health
+  def measure_cluster_health cluster_name, private_ip
+    begin
+      cluster_health = JSON.parse(open(es_url(private_ip, '_cluster', 'health')).read)
+    rescue SocketError, OpenURI::HTTPError, JSON::ParserError, Errno::ECONNREFUSED => e
+      # This node may not be running a webnode...
+      return false
+    end
+    write({
+            :hostname    => "#{cluster_name}-elasticsearch",
+            :application => "_cluster",                 
+            :templates   => self.class::CLUSTER_TEMPLATES,
+            :hostgroup   => self.class::HOSTGROUP
+          }) do |d|
+      d << ['status',              cluster_health['status']               ]
+      d << ['nodes.total',         cluster_health['number_of_nodes']      ]
+      d << ['nodes.data',          cluster_health['number_of_data_nodes'] ]
+      d << ['shards.active',       cluster_health['active_shards']        ]
+      d << ['shards.relocating',   cluster_health['relocating_shards']    ]
+      d << ['shards.unassigned',   cluster_health['unassigned_shards']    ]
+      d << ['shards.initializing', cluster_health['initializing_shards']  ]
+    end
+    true
+  end
+
+  def measure_cluster_indices cluster_name, private_ip
+    begin
+      index_data = JSON.parse(open(es_url(private_ip, '_status')).read)
+    rescue SocketError, OpenURI::HTTPError, JSON::ParserError, Errno::ECONNREFUSED => e
+      # This node may not be running a webnode...
+      return false
+    end
+    index_data['indices'].each_pair do |index_name, index_data|
+      write({
+              :hostname   => "#{cluster_name}-elasticsearch",
+              :appliation => index_name,
+              :templates  => self.class::CLUSTER_TEMPLATES,
+              :hostgroup  => self.class::HOSTGROUP
+            }) do |d|
+        d << ["#{index_name}.size",           index_data["index"]["size_in_bytes"] ]
+        d << ["#{index_name}.docs.num",       index_data["docs"]["num_docs"]       ]
+        d << ["#{index_name}.docs.max",       index_data["docs"]["max_doc"]        ]
+        d << ["#{index_name}.docs.deleted",   index_data["docs"]["deleted_docs"]   ]
+        d << ["#{index_name}.operations",     index_data["translog"]["operations"] ]
+        d << ["#{index_name}.merges.total",   index_data["merges"]["total"]        ]
+        d << ["#{index_name}.merges.current", index_data["merges"]["current"]      ]
+      end
+    end
+    true
+  end
+
+  def measure_cluster_nodes cluster_name, private_ip
+    begin
+      nodes_data       = JSON.parse(open(es_url(private_ip, '_cluster', 'nodes')).read)
+      nodes_stats_data = JSON.parse(open(es_url(private_ip, '_cluster', 'nodes', 'stats')).read)
+    rescue SocketError, OpenURI::HTTPError, JSON::ParserError, Errno::ECONNREFUSED => e
+      # This node may not be running a webnode...
+      return false
+    end
+
+    nodes_stats_data['nodes'].each_pair do |id, stats|
+
+      ip        = nodes_data['nodes'][id]['network']['primary_interface']['address']
+      node_name = chef_node_name_from_ip(ip)
+      next unless node_name
+      write({
+              :hostname    => node_name,
+              :application => "Elasticsearch",
+              :templates   => self.class::NODE_TEMPLATES
+            }) do |d|
+        # concurrency
+        d << ['es.jvm.threads.count',     stats['jvm']['threads']['count']                   ]
+
+        # garbage collection
+        d << ['es.jvm.gc.coll_time',      stats['jvm']['gc']['collection_time_in_millis']    ]
+        d << ['es.jvm.gc.coll_count',     stats['jvm']['gc']['collection_count']             ]
+
+        # memory
+        d << ['es.jvm.mem.heap_used',     stats['jvm']['mem']['heap_used_in_bytes']          ]
+        d << ['es.jvm.mem.non_heap_used', stats['jvm']['mem']['non_heap_used_in_bytes']      ]
+        d << ['es.jvm.mem.heap_comm',     stats['jvm']['mem']['heap_committed_in_bytes']     ]
+        d << ['es.jvm.mem.non_heap_comm', stats['jvm']['mem']['non_heap_committed_in_bytes'] ]
+
+        # indices
+        d << ['es.indices.size',          stats['indices']['size_in_bytes']                  ]
+      end
+    end
+    true
+  end
+
+end
+
+ESMonitor.run if $0 == __FILE__

data/lib/rubix/examples/hbase_monitor.rb

@@ -0,0 +1,87 @@
+#!/usr/bin/env ruby
+
+require 'zabbix_cluster_monitor'
+require 'net/http'
+require 'crack'
+
+class HBaseMonitor < ZabbixClusterMonitor
+
+  # Hostgroups for clusters & hosts that need to be created.
+  CLUSTER_HOSTGROUP = 'HBase clusters'
+
+  # Templates for any hosts that need to be created.
+  CLUSTER_TEMPLATES = 'Template_HBase_Cluster'
+  NODE_TEMPLATES    = 'Template_HBase_Node'
+
+  def matching_chef_nodes
+    Chef::Search::Query.new.search('node', 'provides_service:*hbase-stargate AND facet_name:alpha')
+  end
+  
+  def measure_cluster cluster_name
+    measured_cluster_status  = false
+    private_ips_by_cluster[cluster_name].each do |private_ip|
+      measured_cluster_status  = measure_cluster_status(cluster_name, private_ip)  unless measured_cluster_status
+      break if measured_cluster_status
+    end
+  end
+
+  # Measure the cluster health metrics -- /status/cluster
+  def measure_cluster_status cluster_name, private_ip
+    begin
+      connection = Net::HTTP.new(private_ip, 8080) # FIXME port
+      request    = Net::HTTP::Get.new('/status/cluster', 'Accept' => 'text/xml')
+      response   = connection.request(request)
+      return false unless response.code.to_i == 200
+      
+      data           = Crack::XML.parse(response.body)
+      cluster_status = data['ClusterStatus']
+      dead_nodes     = cluster_status['DeadNodes'] ? cluster_status['DeadNodes']['Node'] : []
+      live_nodes     = cluster_status['LiveNodes']['Node']
+    rescue NoMethodError, SocketError, REXML::ParseException, Errno::ECONNREFUSED => e
+      # puts "#{e.class} -- #{e.message}"
+      # puts e.backtrace
+      return false
+    end
+
+    write({
+            :hostname    => "#{cluster_name}-hbase",
+            :application => '_cluster',
+            :hostgroup   => 'HBase Clusters',
+            :templates   => self.class::CLUSTER_TEMPLATES
+          }) do |d|
+      d << ['requests',    cluster_status['requests']]
+      d << ['regions',     cluster_status['regions']]
+      d << ['load',        cluster_status['averageLoad']]
+      d << ['nodes.dead',  dead_nodes.size]
+      d << ['nodes.alive', live_nodes.size]
+    end
+    measure_cluster_tables(cluster_name, data)
+    measure_cluster_nodes(cluster_name, live_nodes)
+    true
+  end
+
+  def measure_cluster_tables cluster_name, data
+    # FIXME...not sure how best to get information about "tables" in HBase...
+  end
+
+  def measure_cluster_nodes cluster_name, live_nodes
+    live_nodes.each do |live_node|
+      next unless live_node
+      ip        = (live_node['name'] || '').split(':').first
+      node_name = chef_node_name_from_ip(ip)
+      next unless node_name
+      write({
+              :hostname    => node_name,
+              :application => "HBase",
+              :templates   => self.class::NODE_TEMPLATES
+            }) do |d|
+        d << ['hbase.regions',   (live_node['Region'] || []).size]
+        d << ['hbase.heap_size', live_node['heapSizeMB']]
+        d << ['hbase.requests',  live_node['requests']]
+      end
+    end
+  end
+
+end
+
+HBaseMonitor.run if $0 == __FILE__

data/lib/rubix/examples/mongo_monitor.rb

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+
+require 'zabbix_cluster_monitor'
+require 'open-uri'
+require 'set'
+require 'mongo'
+
+class MongoMonitor < ZabbixClusterMonitor
+  
+  # Hostgroup for any hosts that needs to be created.
+  HOSTGROUP = 'MongoDB clusters'
+
+  # Templates for any hosts that need to be created.
+  TEMPLATES = 'Template_MongoDB'
+
+  # Names of database to ignore when we find them.
+  IGNORED_DATABASES = %w[db test admin local].to_set
+
+  def matching_chef_nodes
+    Chef::Search::Query.new.search('node', 'provides_service:*-mongodb-server')
+  end
+  
+  def measure_cluster cluster_name
+    measured_mongo_server    = false
+    measured_mongo_databases = false
+    private_ips_by_cluster[cluster_name].each do |private_ip|
+      begin
+        connection = Mongo::Connection.new(private_ip)
+      rescue Mongo::ConnectionFailure => e
+        next
+      end
+      measured_mongo_server    = measure_mongo_server(cluster_name, connection)    unless measured_mongo_server
+      measured_mongo_databases = measure_mongo_databases(cluster_name, connection) unless measured_mongo_databases
+      break if measured_mongo_server && measured_mongo_databases
+    end
+  end
+
+  def measure_mongo_server cluster_name, connection
+    initial = nil, final = nil
+    db       = connection.db('system') # the name of this db doesn't matter?
+    command = {:serverStatus => true}  # the value of the 'serverStatus' key doesn't matter?
+
+    # gather metrics with a 1.0 second gap
+    initial = db.command(command) ; sleep 1.0 ; final = db.command(command)
+    return false unless initial && final
+    dt = final['localTime'].to_f - initial['localTime'].to_f
+
+    write({
+             :hostname    => "#{cluster_name}-mongodb",
+             :application => '_cluster',
+             :hostgroup   => self.class::HOSTGROUP,
+             :templates   => self.class::TEMPLATES
+           }) do |d|
+      
+      # operations
+      d << ['inserts',       (final['opcounters']['insert']  - initial['opcounters']['insert'])  / dt]
+      d << ['queries',       (final['opcounters']['query']   - initial['opcounters']['query'])   / dt]
+      d << ['updates',       (final['opcounters']['update']  - initial['opcounters']['update'])  / dt]
+      d << ['deletes',       (final['opcounters']['delete']  - initial['opcounters']['delete'])  / dt]
+      d << ['getmores',      (final['opcounters']['getmore'] - initial['opcounters']['getmore']) / dt]
+      d << ['commands',      (final['opcounters']['command'] - initial['opcounters']['command']) / dt]
+
+      # memory
+      d << ['mem.resident',  final['mem']['resident']]
+      d << ['mem.virtual',   final['mem']['virtual']]
+      d << ['mem.mapped',    final['mem']['mapped']]
+
+      # disk
+      d << ['flushes',       (final['backgroundFlushing']['flushes']  - initial['backgroundFlushing']['flushes'])  / dt]
+      d << ['flush_time',    (final['backgroundFlushing']['total_ms'] - initial['backgroundFlushing']['total_ms'])     ]
+      d << ['faults',        (final['extra_info']['page_faults']      - initial['extra_info']['page_faults'])      / dt]
+
+      # index
+      d << ['accesses',        (final['indexCounters']['btree']['accesses'] - initial['indexCounters']['btree']['accesses']) / dt]
+      d << ['hits',            (final['indexCounters']['btree']['hits']     - initial['indexCounters']['btree']['hits'])     / dt]
+      d << ['misses',          (final['indexCounters']['btree']['misses']   - initial['indexCounters']['btree']['misses'])   / dt]
+      d << ['resets',          (final['indexCounters']['btree']['resets']   - initial['indexCounters']['btree']['resets'])   / dt]
+
+      # read/write load
+      d << ['queue.total',   final['globalLock']['currentQueue']['total']]
+      d << ['queue.read',    final['globalLock']['currentQueue']['readers']]
+      d << ['queue.write',   final['globalLock']['currentQueue']['writers']]
+      d << ['clients.total', final['globalLock']['activeClients']['total']]
+      d << ['clients.read',  final['globalLock']['activeClients']['readers']]
+      d << ['clients.write', final['globalLock']['activeClients']['writers']]
+
+      # network
+      d << ['net.in',        (final['network']['bytesIn']     - initial['network']['bytesIn'])     / dt]
+      d << ['net.out',       (final['network']['bytesOut']    - initial['network']['bytesOut'])    / dt]
+      d << ['requests',      (final['network']['numRequests'] - initial['network']['numRequests']) / dt]
+      d << ['connections',   final['connections']['current']]
+    end
+    true
+  end
+
+  def measure_mongo_databases cluster_name, connection
+    dbs = connection.database_names
+    return true if dbs.size == 0 # nothing to do here
+
+    dbs.each do |database_name|
+      next if self.class::IGNORED_DATABASES.include?(database_name.downcase)
+      stats = connection.db(database_name).stats()
+
+      write({
+               :hostname    => "#{cluster_name}-mongodb",
+               :application => database_name,
+               :hostgroup   => self.class::HOSTGROUP,
+               :templates   => self.class::TEMPLATES
+             }) do |d|
+        d << ["#{database_name}.collections",      stats["collections"] ]
+        d << ["#{database_name}.objects.count",    stats["objects"]     ]
+        d << ["#{database_name}.objects.avg_size", stats["avgObjSize"]  ]
+        d << ["#{database_name}.size.data",        stats["dataSize"]    ]
+        d << ["#{database_name}.size.disk",        stats["storageSize"] ]
+        d << ["#{database_name}.size.indexes",     stats["indexSize"]   ]
+        d << ["#{database_name}.size.file",        stats["fileSize"]    ]
+        d << ["#{database_name}.extents",          stats["numExtents"]  ]
+        d << ["#{database_name}.indexes",          stats["indexes"]     ]
+      end
+    end
+    true
+  end
+end
+
+MongoMonitor.run if $0 == __FILE__