metrix 0.0.1 → 0.0.2

data/bin/metrix

@@ -1,68 +1,5 @@
 #!/usr/bin/env ruby
 $:.push(File.expand_path("../../lib", __FILE__))
-require "time"
-require "metrix"
-require "metrix/elastic_search"
-require "metrix/graphite"
-require "metrix/mongodb"
-require "syslog/logger"
-require "optparse"
+require "metrix/cli"
 
-Metrix.logger = Syslog::Logger.new("metrix")
-
-metrics = []
-interval = 10
-graphite_host = nil
-opts = OptionParser.new do |o|
-  o.on("--mongodb") do
-    metrics << :mongodb
-  end
-
-  o.on("--es") do
-    metrics << :es
-  end
-
-  o.on("--graphite-host HOST") do |value|
-    graphite_host = value
-  end
-end
-
-opts.parse(ARGV)
-
-if graphite_host.nil?
-  puts "ERROR: graphite-host is required"
-  abort opts.to_s
-end
-
-started = Time.now
-graphite = Graphite.new(graphite_host, 2003)
-cnt = 0
-while true
-  begin
-    cnt += 1
-    now = Time.now.utc
-    if metrics.include?(:es)
-      Metrix.logger.info "fetching elasticsearch metrix"
-      Metrix::ElasticSearch.from_uri("http://127.0.0.1:9200/_status").metrics.each do |k, v|
-        graphite << "elastic_search.#{k} #{v} #{now.to_i}"
-      end
-    end
-
-    if metrics.include?(:mongodb)
-      Metrix.logger.info "fetching mongodb metrix"
-      Metrix::Mongodb.from_uri("http://127.0.0.1:28017/serverStatus").metrics.each do |k, v|
-        graphite << "mongodb.#{k} #{v} #{now.to_i}"
-      end
-    end
-    graphite.flush
-    diff = Time.now - now
-    Metrix.logger.info "finished run in %.06f" % [diff]
-  rescue SystemExit
-    exit
-  rescue => err
-    logger.error "#{err.message}"
-    logger.error "#{err.backtrace.inspect}"
-  ensure
-    sleep interval - (Time.now - started - cnt * interval)
-  end
-end
+Metrix::CLI.new(ARGV).run

data/lib/metrix/base.rb

@@ -1,49 +1,42 @@
 require "net/http"
 require "json"
+require "metrix/metric"
 
-class Base
-  attr_reader :attributes
+module Metrix
+  class Base
+    attr_reader :attributes, :time
 
-  class << self
-    def from_json(json)
-      self.new(JSON.load(json))
-    end
+    class << self
+      def ignore_metrics(*metrics)
+        @ignore = [metrics].flatten
+      end
 
-    def from_uri(uri)
-      from_json(Net::HTTP.get(URI(uri)))
+      def ignore
+        @ignore ||= []
+      end
     end
 
-    def ignore_metrics(*metrics)
-      @ignore = [metrics].flatten
+    def initialize(raw, time = Time.now)
+      @raw = raw
+      @time = time
     end
 
-    def ignore
-      @ignore ||= []
+    def metrics
+      unfiltered_metrics.reject { |k, v| ignore_metric?(k) }.map do |k, v|
+        Metric.new("#{prefix}.#{k}", v, @time, tags)
+      end
     end
-  end
 
-  def initialize(attributes)
-    @attributes = attributes
-  end
+    def tags
+      {}
+    end
 
-  def metrics
-    map_attributes(attributes).reject { |k, v| self.class.ignore.include?(k) }
-  end
+    def ignore_metric?(metric)
+      self.class.ignore.include?(metric)
+    end
 
-  def map_attributes(attributes, prefix = nil)
-    attributes.inject({}) do |hash, (k, v)|
-      path = [prefix, k].compact.join(".")
-      case v
-      when Hash
-        hash.merge!(map_attributes(v, path))
-      when Array
-        v.each_with_index do |array_value, i|
-          hash.merge!(map_attributes(v, "#{path}i"))
-        end
-      when Numeric
-        hash[path] = v
-      end
-      hash
+    def unfiltered_metrics
+      extract(attributes)
     end
   end
 end

data/lib/metrix/cli.rb

@@ -0,0 +1,141 @@
+require "metrix"
+require "metrix/elastic_search"
+require "metrix/mongodb"
+require "metrix/system"
+require "metrix/load"
+require "logger"
+
+module Metrix
+  class CLI
+    attr_reader :reporter, :elastic_search_host, :mongodb_host, :interval
+
+    def initialize(args)
+      @args = args
+      @system = false
+      @interval = 10
+      require "syslog/logger"
+      Metrix.logger = Syslog::Logger.new("metrix")
+    end
+
+    def run
+      opts.parse(@args)
+      Metrix.logger.level = log_level
+      if self.reporter.nil?
+        puts "ERROR: at least one reporter must be specified"
+        abort opts.to_s
+      end
+      cnt = -1
+      started = Time.now
+      while true
+        begin
+          cnt += 1
+          now = Time.now.utc
+          reporter << Metrix::ElasticSearch.new(elastic_search_status)  if elastic_search?
+          reporter << Metrix::Mongodb.new(mongodb_status)               if mongodb?
+          if system?
+            reporter << Metrix::System.new(File.read("/proc/stat"))
+            reporter << Metrix::Load.new(File.read("/proc/loadavg"))
+          end
+
+          if processes?
+            Metrix::Process.all.each do |m|
+              reporter << m
+            end
+          end
+          reporter.flush
+        rescue SystemExit
+          exit
+        rescue => err
+          Metrix.logger.error "#{err.message}"
+          Metrix.logger.error "#{err.backtrace.inspect}"
+        ensure
+          sleep_for = @interval - (Time.now - started - cnt * interval)
+          Metrix.logger.info "finished run in %.06f, sleeping for %.06f" % [Time.now - now, sleep_for]
+          sleep sleep_for
+        end
+      end
+    end
+
+    def log_level
+      @log_level || Logger::INFO
+    end
+
+    def processes?
+      !!@processes
+    end
+
+    def elastic_search?
+      !!@elastic_search
+    end
+
+    def mongodb?
+      !!@mongodb
+    end
+
+    def elastic_search_status
+      Metrix.logger.info "fetching elasticsearch metrix"
+      Net::HTTP.get(URI("http://127.0.0.1:9200/_status"))
+    end
+
+    def mongodb_status
+      Metrix.logger.info "fetching mongodb metrix"
+      Net::HTTP.get(URI("http://127.0.0.1:28017/serverStatus"))
+    end
+
+    def system?
+      !!@system
+    end
+
+    def log_to_stdout
+      Metrix.logger = Logger.new(STDOUT)
+      Metrix.logger.level = Logger::INFO
+    end
+
+    def opts
+      require "optparse"
+      @opts ||= OptionParser.new do |o|
+        o.on("--mongodb") do
+          @mongodb = true
+        end
+
+        o.on("--elasticsearch") do
+          @elastic_search = true
+        end
+
+        o.on("--graphite-host HOST") do |value|
+          require "metrix/graphite"
+          @reporter = Metrix::Graphite.new(value, 2003)
+        end
+
+        o.on("--opentsdb-host HOST") do |value|
+          require "metrix/opentsdb"
+          @reporter = Metrix::OpenTSDB.new(value, 4242)
+        end
+
+        o.on("--stdout") do
+          require "metrix/reporter/stdout"
+          @reporter = Metrix::Reporter::Stdout.new
+          log_to_stdout
+        end
+
+        o.on("--debug") do
+          @log_level = Logger::DEBUG
+        end
+
+        o.on("--no-syslog") do
+          log_to_stdout
+        end
+
+        o.on("--processes") do
+          require "metrix/process"
+          @processes = true
+        end
+
+        o.on("--system") do
+          require "metrix/system"
+          @system = true
+        end
+      end
+    end
+  end
+end

data/lib/metrix/elastic_search.rb

@@ -1,7 +1,11 @@
-require "metrix/base"
+require "metrix/json"
 
 module Metrix
-  class ElasticSearch < Base
+  class ElasticSearch < Json
     ignore_metrics []
+
+    def prefix
+      "elasticsearch"
+    end
   end
 end

data/lib/metrix/graphite.rb

@@ -1,41 +1,42 @@
 require "socket"
+require "metrix"
 
-class Graphite
-  def initialize(host, port = 2003)
-    @host = host
-    @port = port
-  end
-
-  def <<(m)
-    metrics << m
-    flush if metrics.count > 90
-  end
+module Metrix
+  class Graphite
+    attr_reader :host, :port
 
-  def flush
-    if metrics.empty?
-      logger.info "nothing to send"
-      return
+    def initialize(host, port = 2003)
+      @host = host
+      @port = port
     end
-    started = Time.now
-    Socket.tcp(@host, @port) do |socket|
-      metrics.each do |m|
-        logger.debug "sending #{m}"
-        socket.puts "metrix.#{hostname}.#{m}"
+
+    def <<(metric)
+      metric.metrics.each do |m|
+        logger.debug "adding #{m.to_graphite}"
+        buffers << m.to_graphite
+        flush if buffers.count > 90
       end
     end
-    logger.info "sent #{metrics.count} in %.06fs" % [Time.now - started]
-    metrics.clear
-  end
 
-  def hostname
-    @hostname ||= `hostname`.strip
-  end
+    def buffers
+      @buffers ||= []
+    end
 
-  def metrics
-    @metrics ||= []
-  end
+    def flush
+      if buffers.empty?
+        logger.info "nothing to send"
+        return
+      end
+      started = Time.now
+      Socket.tcp(@host, @port) do |socket|
+        socket.puts(buffers.join("\n"))
+      end
+      logger.info "sent #{buffers.count} in %.06fs" % [Time.now - started]
+      buffers.clear
+    end
 
-  def logger
-    Metrix.logger
+    def logger
+      Metrix.logger
+    end
   end
 end

data/lib/metrix/json.rb

@@ -0,0 +1,26 @@
+require "metrix/base"
+
+module Metrix
+  class Json < Base
+    def attributes
+      @attributes ||= JSON.load(@raw)
+    end
+
+    def extract(attributes, prefix = nil)
+      attributes.inject({}) do |hash, (k, v)|
+        path = [prefix, k].compact.join(".")
+        case v
+        when Hash
+          hash.merge!(extract(v, path))
+        when Array
+          v.each_with_index do |array_value, i|
+            hash.merge!(extract(v, "#{path}i"))
+          end
+        when Numeric
+          hash[path] = v
+        end
+        hash
+      end
+    end
+  end
+end

data/lib/metrix/load.rb

@@ -0,0 +1,38 @@
+require "metrix/base"
+
+module Metrix
+  class Load < Base
+    def initialize(data)
+      @data = data
+      @time = Time.now
+    end
+
+    def extract(data)
+      {
+        load1: load1,
+        load5: load5,
+        load15: load15,
+      }
+    end
+
+    def prefix
+      "system.load"
+    end
+
+    def load15
+      chunks.at(2).to_f
+    end
+
+    def load5
+      chunks.at(1).to_f
+    end
+
+    def load1
+      chunks.at(0).to_f
+    end
+
+    def chunks
+      @chunks ||= @data.split(" ")
+    end
+  end
+end

data/lib/metrix/metric.rb

@@ -0,0 +1,37 @@
+require "metrix"
+
+module Metrix
+  class Metric
+    attr_reader :key, :value, :time, :tags
+
+    def initialize(key, value, time, tags = {})
+      @key = key
+      @value = value
+      @time = time
+      @tags = tags
+    end
+
+    def to_opentsdb
+      chunks = [:put, key, time.utc.to_i, value]
+      tags.merge(hostname: Metrix.hostname, database: database).each do |k, v|
+        chunks << "#{k}=#{v}" if v
+      end
+      chunks.join(" ")
+    end
+
+    def to_graphite
+      chunks = [graphite_prefix]
+      chunks << "databases.#{database}" if database
+      chunks << key
+      "#{chunks.join(".")} #{value} #{time.utc.to_i}"
+    end
+
+    def database
+      tags[:database]
+    end
+
+    def graphite_prefix
+      "metrix.#{Metrix.hostname}"
+    end
+  end
+end

data/lib/metrix/mongodb.rb

@@ -1,7 +1,38 @@
-require "metrix/base"
+require "metrix/json"
+require "metrix/metric"
 
 module Metrix
-  class Mongodb < Base
+  class Mongodb < Json
     ignore_metrics %w(ok mem.bits pid uptime uptimeMillis uptimeEstimate)
+
+    def prefix
+      "mongodb"
+    end
+
+    DATABASE_RECORD_SET = /^recordStats\.(.*?)\.(.*)/
+    DATABASE_LOCK = /^locks/
+
+    def ignore_metric?(metric)
+      metric[DATABASE_RECORD_SET] ||
+      metric[DATABASE_LOCK] ||
+      super
+    end
+
+    def tagged_metrics
+      unfiltered_metrics.map do |k, v|
+        if k.match(DATABASE_RECORD_SET)
+          database = $1
+          Metric.new($2, "#{prefix}.locks.#{v}", time, database: database)
+        elsif k.match(DATABASE_LOCK)
+          chunks = k.split(".")
+          offset = 0
+          offset = 1 if chunks.at(1) == "" # for "." database
+          database = chunks.at(1 + offset)
+          metric = chunks[(2 + offset)..-1].join(".")
+          database = "." if database == ""
+          Metric.new("#{prefix}.recordStats.#{metric}", v, time, database: database)
+        end
+      end.compact
+    end
   end
 end

data/lib/metrix/opentsdb.rb

@@ -0,0 +1,33 @@
+require "socket"
+
+module Metrix
+  class OpenTSDB
+    def initialize(host, port = 4242)
+      @host = host
+      @port = port
+    end
+
+    def <<(metric)
+      metric.metrics.each do |m|
+        Metrix.logger.debug "buffering #{m.to_opentsdb}"
+        buffers << m.to_opentsdb
+        flush if buffers.count >= 90
+      end
+    rescue => err
+      Metrix.logger.error "#{err.message} #{err.inspect}"
+    end
+
+    def flush
+      return if buffers.empty?
+      Metrix.logger.info "sending #{buffers.count} to #{@host}:#{@port}"
+      Socket.tcp(@host, @port) do |socket|
+        socket.puts buffers.join("\n")
+      end
+      buffers.clear
+    end
+
+    def buffers
+      @buffers ||= []
+    end
+  end
+end

data/lib/metrix/process.rb

@@ -0,0 +1,94 @@
+require "metrix/base"
+
+module Metrix
+  class Process < Base
+    attr_reader :time
+
+    class << self
+      def all
+        Dir.glob("/proc/*").select do |path|
+          File.directory?(path) && File.basename(path)[/^\d+$/]
+        end.map do |path|
+          Metrix::Process.new(File.read(path + "/stat"))
+        end
+      end
+    end
+
+    def name
+      comm.gsub(/^\(/, "").gsub(/\)$/, "")
+    end
+
+    def chunks
+      @chunks ||= @raw.split(" ").map { |c| cast_int(c) }
+    end
+
+    def tags
+      {
+        name:   name,
+        pid:    pid,
+        ppid:   ppid,
+      }
+    end
+
+    def unfiltered_metrics
+      {
+        minflt: minflt,
+        cminflt: cminflt,
+        majflt: majflt,
+        cmajflt: cmajflt,
+        utime: utime,
+        stime: stime,
+        cutime: cutime,
+        sctime: sctime,
+        num_threads: num_threads,
+        vsize: vsize,
+        rss: rss,
+      }
+    end
+
+    def prefix
+      "system.process"
+    end
+
+    def cast_int(str)
+      Integer(str) rescue str
+    end
+
+    {
+      :pid => 0,
+      :comm => 1,
+      :state => 2,
+      :ppid => 3,
+      :pgrp => 4,
+      :session => 5,
+      :tty_nr => 6,
+      :tpgid => 7,
+      :flags => 8,
+      :minflt => 9,
+      :cminflt => 10,
+      :majflt => 11,
+      :cmajflt => 12,
+      :utime => 13,
+      :stime => 14,
+      :cutime => 15,
+      :sctime => 16,
+      :priority => 17,
+      :nice => 18,
+      :num_threads => 19,
+      :itrealvalue => 20,
+      :starttime => 21,
+      :vsize => 22,
+      :rss => 23,
+      :rsslim => 24,
+      :startcode => 25,
+      :endcode => 26,
+      :startstac => 27,
+      :guest_time => 42,
+      :cguest_time => 43,
+    }.each do |k, v|
+      define_method(k) do
+        chunks.at(v)
+      end
+    end
+  end
+end

data/lib/metrix/reporter/stdout.rb

@@ -0,0 +1,17 @@
+module Metrix
+  module Reporter
+    class Stdout
+      def initialize
+      end
+
+      def <<(metric)
+        metric.metrics.each do |m|
+          Metrix.logger.info "#{m.key} #{m.value} #{m.tags.inspect}"
+        end
+      end
+
+      def flush
+      end
+    end
+  end
+end