metrix 0.0.7 → 0.0.8

data/README.md

@@ -6,20 +6,26 @@ TODO: Write a gem description
 
 Add this line to your application's Gemfile:
 
-    gem 'metrix'
+    gem install 'metrix'
 
 ## Configuration
 
-    #/etc/metrix.tml
+    # in /etc/metrix.tml
+
+    # local metrics
+    load:           true
+    system:         true
+    processes:      true
+
+    # http metrics
     elasticsearch:  http://127.0.0.1:9200/_status
     mongodb:        http://127.0.0.1:28017/serverStatus
     fpm:            http://127.0.0.1:9001/fpm-status
     nginx:          http://127.0.0.1:8000/
+
+    # reporters
     opentsdb:       tcp://127.0.0.1:4242/
     graphite:       tcp://127.0.0.1:2003/
-    load:           true
-    system:         true
-    processes:      true
 
 ## Start
 

data/lib/metrix.rb

@@ -13,5 +13,18 @@ module Metrix
     def hostname
       @hostname ||= `hostname`.strip
     end
+
+    def known_metrics
+      Dir.glob(File.expand_path("../metrix/*.rb", __FILE__)).each do |path|
+        require path
+      end
+      Base.subclasses.map do |clazz|
+        raise "known_metrics not set for #{clazz}" if clazz.known_metrics.nil?
+        raise "prefix not set for #{clazz}" if clazz.prefix.nil?
+        clazz.known_metrics.map do |m|
+          "#{clazz.prefix}.#{m}"
+        end
+      end.flatten.compact.sort
+    end
   end
 end

data/lib/metrix/base.rb

@@ -11,9 +11,33 @@ module Metrix
         @ignore = [metrics].flatten
       end
 
+      def inherited(clazz)
+        subclasses << clazz
+      end
+
+      def set_prefix(prefix)
+        @prefix = prefix
+      end
+
+      def prefix
+        @prefix
+      end
+
+      def subclasses
+        @subclasses ||= []
+      end
+
       def ignore
         @ignore ||= []
       end
+
+      def set_known_metrics(*metrics)
+        @known_metrics = metrics.flatten
+      end
+
+      def known_metrics
+        @known_metrics
+      end
     end
 
     def initialize(raw, time = Time.now)
@@ -24,7 +48,15 @@ module Metrix
     def metrics
       unfiltered_metrics.reject { |k, v| ignore_metric?(k) }.map do |k, v|
         Metric.new("#{prefix}.#{k}", v, @time, tags)
-      end
+      end + tagged_metrics
+    end
+
+    def tagged_metrics
+      []
+    end
+
+    def prefix
+      self.class.prefix
     end
 
     def tags

data/lib/metrix/cli.rb

@@ -5,7 +5,7 @@ require "metrix/nginx"
 require "metrix/system"
 require "metrix/load"
 require "metrix/fpm"
-require "metrix/process"
+require "metrix/process_metric"
 require "metrix/load"
 require "logger"
 require "fileutils"
@@ -32,9 +32,9 @@ module Metrix
 
     def run
       parse!
-      load_configs_from_file!
       case @action
       when "start"
+        load_configs_from_file!
         if running?
           logger.warn "refuse to run. seems that #{pid_path} exists!"
           abort "not allowed to run" if running?
@@ -65,7 +65,10 @@ module Metrix
         system "kill #{pid}"
         puts "killed #{pid}"
       when "configtest"
+        load_configs_from_file!
         puts "running configtest #{attributes.inspect}"
+      when "list_metrics"
+        puts Metrix.known_metrics.join("\n")
       else
         logger.warn "action #{action} unknown!"
         abort "action #{action} unknown!"

data/lib/metrix/elastic_search.rb

@@ -1,11 +1,28 @@
-require "metrix/json"
+require "metrix/json_metric"
 
 module Metrix
-  class ElasticSearch < Json
-    ignore_metrics []
+  class ElasticSearch < Base
+    include JsonMetric
+    set_prefix "elasticsearch"
+    set_known_metrics %w(
+      _shards.total _shards.successful _shards.failed index.primary_size_in_bytes
+      index.size_in_bytes translog.operations docs.num_docs docs.max_doc
+      docs.deleted_docs merges.current merges.current_docs merges.current_size_in_bytes
+      merges.total merges.total_time_in_millis merges.total_docs
+      merges.total_size_in_bytes refresh.total refresh.total_time_in_millis flush.total
+      flush.total_time_in_millis
+    )
 
-    def prefix
-      "elasticsearch"
+    DATABASE_INDEX = /^indices\./
+    def metrics
+      unfiltered_metrics.map do |k, v|
+        if k.match(DATABASE_INDEX)
+          _, index_name, key = k.split(".", 3)
+          Metric.new("#{prefix}.#{key}", v, time, index: index_name)
+        else
+          Metric.new("#{prefix}.#{k}", v, time)
+        end
+      end.compact
     end
   end
 end

data/lib/metrix/fpm.rb

@@ -2,6 +2,13 @@ require "metrix/base"
 
 module Metrix
   class FPM < Base
+    set_prefix "fpm"
+    set_known_metrics %w(
+      accepted_conn start_since accepted_conn listen_queue max_listen_queue
+      listen_queue_len idle_processes active_processes total_processes max_active_processes
+      max_children_reached slow_requests
+    )
+
     def initialize(data)
       @data = data
       @time = Time.now

data/lib/metrix/{json.rb → json_metric.rb}

@@ -1,7 +1,7 @@
 require "metrix/base"
 
 module Metrix
-  class Json < Base
+  module JsonMetric
     def attributes
       @attributes ||= JSON.load(@raw)
     end

data/lib/metrix/load.rb

@@ -2,6 +2,9 @@ require "metrix/base"
 
 module Metrix
   class Load < Base
+    set_prefix "system.load"
+    set_known_metrics %w(load1 load5 load15)
+
     def initialize(data)
       @data = data
       @time = Time.now

data/lib/metrix/mongodb.rb

@@ -1,28 +1,45 @@
-require "metrix/json"
+require "metrix/json_metric"
 require "metrix/metric"
 
 module Metrix
-  class Mongodb < Json
-    ignore_metrics %w(ok mem.bits pid uptime uptimeMillis uptimeEstimate)
-
-    def prefix
-      "mongodb"
-    end
+  class Mongodb < Base
+    include JsonMetric
+    set_prefix "mongodb"
+    set_known_metrics %w(
+      asserts.msg asserts.regular asserts.rollovers asserts.user asserts.warning backgroundFlushing.average_ms
+      backgroundFlushing.flushes backgroundFlushing.last_ms backgroundFlushing.total_ms connections.available connections.current
+      connections.totalCreated cursors.clientCursors_size cursors.timedOut cursors.totalOpen dur.commits dur.commitsInWriteLock
+      dur.compression dur.earlyCommits dur.journaledMB dur.timeMs.dt dur.timeMs.prepLogBuffer dur.timeMs.remapPrivateView
+      dur.timeMs.writeToDataFiles dur.timeMs.writeToJournal dur.writeToDataFilesMB extra_info.page_faults globalLock.activeClients.readers
+      globalLock.activeClients.total globalLock.activeClients.writers globalLock.currentQueue.readers globalLock.currentQueue.total
+      globalLock.currentQueue.writers globalLock.lockTime globalLock.totalTime indexCounters.accesses indexCounters.hits
+      indexCounters.missRatio indexCounters.misses indexCounters.resets mem.mapped mem.mappedWithJournal mem.resident mem.virtual
+      metrics.document.deleted metrics.document.inserted metrics.document.returned metrics.document.updated metrics.getLastError.wtime.num
+      metrics.getLastError.wtime.totalMillis metrics.getLastError.wtimeouts metrics.operation.fastmod metrics.operation.idhack
+      metrics.operation.scanAndOrder metrics.queryExecutor.scanned metrics.record.moves metrics.repl.apply.batches.num
+      metrics.repl.apply.batches.totalMillis metrics.repl.apply.ops metrics.repl.buffer.count metrics.repl.buffer.maxSizeBytes
+      metrics.repl.buffer.sizeBytes metrics.repl.network.bytes metrics.repl.network.getmores.num metrics.repl.network.getmores.totalMillis
+      metrics.repl.network.ops metrics.repl.network.readersCreated metrics.repl.oplog.insert.num metrics.repl.oplog.insert.totalMillis
+      metrics.repl.oplog.insertBytes metrics.repl.preload.docs.num metrics.repl.preload.docs.totalMillis metrics.repl.preload.indexes.num
+      metrics.repl.preload.indexes.totalMillis metrics.ttl.deletedDocuments metrics.ttl.passes network.bytesIn network.bytesOut
+      network.numRequests opcounters.command opcounters.delete opcounters.getmore opcounters.insert opcounters.query
+      opcounters.update opcountersRepl.command opcountersRepl.delete opcountersRepl.getmore opcountersRepl.insert opcountersRepl.query
+      opcountersRepl.update recordStats.accessesNotInMemory recordStats.pageFaultExceptionsThrown recordStats.timeAcquiringMicros.R
+      recordStats.timeAcquiringMicros.W recordStats.timeAcquiringMicros.r recordStats.timeAcquiringMicros.r recordStats.timeAcquiringMicros.w
+      recordStats.timeAcquiringMicros.w recordStats.timeLockedMicros.R recordStats.timeLockedMicros.W recordStats.timeLockedMicros.r
+      recordStats.timeLockedMicros.r recordStats.timeLockedMicros.w recordStats.timeLockedMicros.w uptime
+    )
 
     DATABASE_RECORD_SET = /^recordStats\.(.*?)\.(.*)/
     DATABASE_LOCK = /^locks/
 
-    def ignore_metric?(metric)
-      metric[DATABASE_RECORD_SET] ||
-      metric[DATABASE_LOCK] ||
-      super
-    end
-
-    def tagged_metrics
+    def metrics
       unfiltered_metrics.map do |k, v|
+        next unless self.class.known_metrics.include?(k)
         if k.match(DATABASE_RECORD_SET)
+          next if %w(accessesNotInMemory pageFaultExceptionsThrown).include?($2)
           database = $1
-          Metric.new($2, "#{prefix}.locks.#{v}", time, database: database)
+          Metric.new("#{prefix}.#{$2}", v, time, database: database)
         elsif k.match(DATABASE_LOCK)
           chunks = k.split(".")
           offset = 0
@@ -31,6 +48,8 @@ module Metrix
           metric = chunks[(2 + offset)..-1].join(".")
           database = "." if database == ""
           Metric.new("#{prefix}.recordStats.#{metric}", v, time, database: database)
+        else
+          Metric.new("#{prefix}.#{k}", v, time)
         end
       end.compact
     end

data/lib/metrix/nginx.rb

@@ -2,6 +2,9 @@ require "metrix/base"
 
 module Metrix
   class Nginx < Base
+    set_prefix "nginx"
+    set_known_metrics %w(accepts handled requests active_connections reading writing waiting)
+
     def initialize(data)
       @data = data
       @time = Time.now
@@ -17,10 +20,6 @@ module Metrix
       end
     end
 
-    def prefix
-      "nginx"
-    end
-
     def extract(data = nil)
       {
         accepts: accepts,

data/lib/metrix/{process.rb → process_metric.rb}

@@ -3,6 +3,10 @@ require "metrix/base"
 module Metrix
   class ProcessMetric < Base
     attr_reader :time
+    set_known_metrics %w(minflt cminflt majflt cmajflt utime stime cutime sctime num_threads
+      vsize rss
+    )
+    set_prefix "system.process"
 
     class << self
       def all
@@ -46,10 +50,6 @@ module Metrix
       }
     end
 
-    def prefix
-      "system.process"
-    end
-
     def cast_int(str)
       Integer(str) rescue str
     end

data/lib/metrix/system.rb

@@ -2,6 +2,12 @@ require "metrix/base"
 
 module Metrix
   class System < Base
+    set_prefix "system"
+    set_known_metrics %w(
+      processes procs_running procs_blocked ctxt cpu.user cpu.nice cpu.system cpu.idle
+      cpu.iowait cpu.irq cpu.softirq 
+    )
+
     class Cpu
       def initialize(values)
         @values = values
@@ -49,10 +55,6 @@ module Metrix
       }
     end
 
-    def prefix
-      "system"
-    end
-
     def cpu
       Cpu.new(@raw[/^cpu (.*)/, 1].split(" ").map(&:to_i))
     end

data/lib/metrix/version.rb

@@ -1,3 +1,3 @@
 module Metrix
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/spec/lib/metrix/elastic_search_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 require "metrix/elastic_search"
 
-describe "Metrox::ElasticSearch" do
+describe "Metrix::ElasticSearch" do
   let(:payload) { FIXTURES_PATH.join("es_status.json").read }
   subject(:es_status) { Metrix::ElasticSearch.new(payload) }
   it { should_not be_nil }
@@ -9,7 +9,8 @@ describe "Metrox::ElasticSearch" do
   it { subject.time.should_not be_nil }
 
   describe "#metrics" do
-    subject(:metrics) { hash_metrics(es_status.metrics) }
+    subject(:extracted) { hash_metrics(es_status.metrics) }
+
     it { should be_kind_of(Hash) }
     it { should_not be_empty }
 
@@ -17,27 +18,38 @@ describe "Metrox::ElasticSearch" do
       "elasticsearch._shards.total"=>10,
       "elasticsearch._shards.successful"=>5,
       "elasticsearch._shards.failed"=>0,
-      "elasticsearch.indices.search.index.primary_size_in_bytes"=>2008,
-      "elasticsearch.indices.search.index.size_in_bytes"=>2008,
-      "elasticsearch.indices.search.translog.operations"=>1,
-      "elasticsearch.indices.search.docs.num_docs"=>1,
-      "elasticsearch.indices.search.docs.max_doc"=>1,
-      "elasticsearch.indices.search.docs.deleted_docs"=>0,
-      "elasticsearch.indices.search.merges.current"=>0,
-      "elasticsearch.indices.search.merges.current_docs"=>0,
-      "elasticsearch.indices.search.merges.current_size_in_bytes"=>0,
-      "elasticsearch.indices.search.merges.total"=>0,
-      "elasticsearch.indices.search.merges.total_time_in_millis"=>0,
-      "elasticsearch.indices.search.merges.total_docs"=>0,
-      "elasticsearch.indices.search.merges.total_size_in_bytes"=>0,
-      "elasticsearch.indices.search.refresh.total"=>6,
-      "elasticsearch.indices.search.refresh.total_time_in_millis"=>124,
-      "elasticsearch.indices.search.flush.total"=>0,
     }.each do |k, v|
       it "should set attributes #{k.inspect} to #{v.inspect}" do
         subject[k].value.should eq(v)
       end
     end
+
+    it { should_not include("elasticsearch.indices.search.flush.total") }
     it { subject.keys.count.should eq(20) }
   end
+
+  it { subject.unfiltered_metrics.should include("indices.search.flush.total") }
+
+  describe "#metrics" do
+    subject(:tagged_metrics) { es_status.metrics }
+    it { should be_kind_of(Array) }
+    it { should_not be_empty }
+
+    it { subject.count.should eq(20) }
+
+    describe "first metric" do
+
+      before :each do
+        es_status.stub(:time) { Time.at(10) }
+      end
+
+      subject(:first) { tagged_metrics.select { |m| m.key == "elasticsearch.index.primary_size_in_bytes" }.first }
+      it { should_not be_nil }
+      it { subject.key.should start_with("elasticsearch.index.primary_size_in_bytes") }
+      it { subject.value.should eq(2008) }
+      it { subject.time.should_not be_nil }
+      it { subject.tags.should eq(index: "search") }
+      it { subject.to_opentsdb.should eq("put elasticsearch.index.primary_size_in_bytes 10 2008 index=search hostname=test.host") }
+    end
+  end
 end

data/spec/lib/metrix/mongodb_spec.rb

@@ -7,19 +7,15 @@ describe "Metrix::Mongodb" do
   subject(:status) { Metrix::Mongodb.new(payload) }
   it { should_not be_nil }
 
-  describe "tagged_metrics" do
-    subject(:tagged_metrics) { status.tagged_metrics }
-    it { should be_kind_of(Array) }
-    it { should_not be_empty }
-  end
+  describe "metric keys" do
+    subject(:keys) { status.metrics.map(&:key) }
 
-  describe "ignore_metric?" do
     %w(
-      locks...timeAcquiringMicros.W
-      recordStats.opentsdb.pageFaultExceptionsThrown
-    ).each do |m|
-      it "should ignore #{m}" do
-        status.should be_ignore_metric(m)
+      mongodb.ok mongodb.mem.bits mongodb.pid
+      mongodb.uptimeMillis mongodb.uptimeEstimate
+    ).each do |name|
+      it "should not have key #{name.inspect}" do
+        subject.should_not include(name)
       end
     end
   end
@@ -28,17 +24,6 @@ describe "Metrix::Mongodb" do
     subject(:metrics) { hash_metrics(status.metrics) }
     it { should be_kind_of(Hash) }
 
-    %w(
-      mongodb.ok mongodb.mem.bits mongodb.pid mongodb.uptime
-      mongodb.uptimeMillis mongodb.uptimeEstimate
-      mongodb.recordStats.opentsdb.pageFaultExceptionsThrown
-      mongodb.locks.local.timeLockedMicros.r
-    ).each do |name|
-      it "should not have key #{name.inspect}" do
-        subject.should_not have_key(name)
-      end
-    end
-
     {
       "mongodb.globalLock.totalTime"=>474819000,
       "mongodb.globalLock.lockTime"=>1060706,
@@ -49,6 +34,7 @@ describe "Metrix::Mongodb" do
       "mongodb.globalLock.activeClients.readers"=>0,
       "mongodb.globalLock.activeClients.writers"=>0,
       "mongodb.mem.resident"=>9,
+      "mongodb.uptime"=> 475,
       "mongodb.mem.virtual"=>2814,
       "mongodb.recordStats.accessesNotInMemory" => 0,
       "mongodb.recordStats.pageFaultExceptionsThrown" => 0,

data/spec/lib/metrix/{process_spec.rb → process_metric_spec.rb}

@@ -1,5 +1,5 @@
 require "spec_helper"
-require "metrix/process"
+require "metrix/process_metric"
 
 describe "Metrix::ProcessMetric" do
   let(:data) { FIXTURES_PATH.join("proc.26928.txt").read }

data/spec/lib/metrix_spec.rb

@@ -0,0 +1,16 @@
+require "spec_helper"
+
+describe "Metrix" do
+  subject(:metrix) { Metrix }
+  it { should_not be_nil }
+
+  describe "#known_metrics" do
+    subject(:known_metrics) { Metrix.known_metrics }
+    it { should be_kind_of(Array) }
+    it { should_not be_empty }
+    it { subject.count.should eq(170) }
+
+    it { should include("mongodb.uptime") }
+  end
+end
+

data/spec/spec_helper.rb

@@ -20,5 +20,7 @@ def hash_metrics(metrics)
   end
 end
 
+require "metrix/process_metric"
+
 require "pathname"
 FIXTURES_PATH = Pathname.new(File.expand_path("../fixtures", __FILE__))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metrix
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -95,13 +95,13 @@ files:
 - lib/metrix/elastic_search.rb
 - lib/metrix/fpm.rb
 - lib/metrix/graphite.rb
-- lib/metrix/json.rb
+- lib/metrix/json_metric.rb
 - lib/metrix/load.rb
 - lib/metrix/metric.rb
 - lib/metrix/mongodb.rb
 - lib/metrix/nginx.rb
 - lib/metrix/opentsdb.rb
-- lib/metrix/process.rb
+- lib/metrix/process_metric.rb
 - lib/metrix/reporter/stdout.rb
 - lib/metrix/system.rb
 - lib/metrix/version.rb
@@ -115,6 +115,7 @@ files:
 - spec/fixtures/php.fpm.status.txt
 - spec/fixtures/proc.26928.txt
 - spec/fixtures/proc.stat.txt
+- spec/lib/metrix/base_spec.rb
 - spec/lib/metrix/cli_spec.rb
 - spec/lib/metrix/elastic_search_spec.rb
 - spec/lib/metrix/fpm_spec.rb
@@ -124,8 +125,9 @@ files:
 - spec/lib/metrix/mongodb_spec.rb
 - spec/lib/metrix/nginx_spec.rb
 - spec/lib/metrix/opentsdb_spec.rb
-- spec/lib/metrix/process_spec.rb
+- spec/lib/metrix/process_metric_spec.rb
 - spec/lib/metrix/system_spec.rb
+- spec/lib/metrix_spec.rb
 - spec/spec_helper.rb
 homepage: ''
 licenses: []
@@ -161,6 +163,7 @@ test_files:
 - spec/fixtures/php.fpm.status.txt
 - spec/fixtures/proc.26928.txt
 - spec/fixtures/proc.stat.txt
+- spec/lib/metrix/base_spec.rb
 - spec/lib/metrix/cli_spec.rb
 - spec/lib/metrix/elastic_search_spec.rb
 - spec/lib/metrix/fpm_spec.rb
@@ -170,6 +173,7 @@ test_files:
 - spec/lib/metrix/mongodb_spec.rb
 - spec/lib/metrix/nginx_spec.rb
 - spec/lib/metrix/opentsdb_spec.rb
-- spec/lib/metrix/process_spec.rb
+- spec/lib/metrix/process_metric_spec.rb
 - spec/lib/metrix/system_spec.rb
+- spec/lib/metrix_spec.rb
 - spec/spec_helper.rb