mongodb-graphite 0.1.5 → 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 462cf3ab0ee602d4cdacff7c2c1f31b92551cdad
+  data.tar.gz: ecc40e1948680682b9f4ef968764f00657ff25c9
+SHA512:
+  metadata.gz: f5ee6b638028c2d35f662bdf15ce7217b2e19ad1711b17b0406c5c687c557c3304a1750e7146c6b1976d6c6125c034bb2dc81f4269b0f63c41e7078d58e537a7
+  data.tar.gz: be5acfc4bc778b4ce0e894b00c0551621047436bfeee66ec211df1420de237246770e6254f57bde69558b6243ffe128058f87387d2711b376287047207da6693

data/Gemfile.lock

@@ -1,23 +1,19 @@
 PATH
   remote: .
   specs:
-    mongodb-graphite (0.1.0.alpha)
-      graphite (>= 0.2.0)
+    mongodb-graphite (1.0.0)
+      daemons
       mongo (>= 1.5.2)
+      simple-graphite (>= 2.1.0)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    bson (1.5.2)
-    eventmachine (0.12.10)
-    graphite (0.2.0)
-      eventmachine
-      rufus-scheduler
-    mongo (1.5.2)
-      bson (= 1.5.2)
-    rufus-scheduler (2.0.16)
-      tzinfo (>= 0.3.23)
-    tzinfo (0.3.31)
+    bson (1.8.6)
+    daemons (1.1.9)
+    mongo (1.8.6)
+      bson (~> 1.8.6)
+    simple-graphite (2.1.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -1,12 +1,42 @@
 Very simple gem to monitor a mongodb cluster and send data to a graphite server.
 
-Install :
+Usage :
 
-Open the mongodb_graphite file and modify the graphite server variable to your graphite server. For exemple : "mygraphite.mydomain.com:2023"
+You have to set an environment variable MONGODB_GRAPHITE_DIR (or it will use the config dir in the current directory).
+Then write a yml file to contain basic settings:
 
-Add the mongodb_graphite to your crontab, for instance
-* * * * mongodb_graphite "graphite.mydomain:2023"
+```yaml
+log_dir: log
+pid_dir: pids
+graphite: graphite.mydomain:3333
+interval: 3
+instances_file: 'config/instances.rb'
+```
 
+Then in config/instances.rb, you can set instances to monitor:
 
-That's all.
+```ruby
+instance 'mongoD' do
+  host 'onemongo.mydomain'
+  port '27021'
+  prefix_callback (lambda { |stats|
+    process_type = stats['process']
+    "production.onemongo.#{process_type}"
+  } )
+end
 
+instance 'mongoD' do
+  host 'anothermongo.mydomain'
+  port '27017'
+  prefix_callback (lambda { |stats|
+    process_type = stats['process']
+    'test.anothermongo.#{process_type}"
+  } )
+end
+```
+
+Finally, you can use the gem as a service (start, status, stop)
+
+```
+mongodb-graphite start
+```

data/bin/mongodb-graphite

@@ -1,93 +1,42 @@
 #!/usr/bin/env ruby
 
-require 'mongo'
-require 'graphite'
-require 'peach'
-
-graphite_server = ARGV[1]
-prefix_common = ARGV[0]+"."
-puts graphite_server
-g = Graphite::Logger.new(graphite_server)
-
-to_ignore =[ "set", "repl", "time","qr|qw","ar|aw","conn"]
-to_convert = ["mapped","vsize","res","netIn","netOut"]
-output_average = ["mapped","vsize","res","locked","idxmiss"]
-output_sum = ["netIn", "netOut","insert","query","update","delete","getmore","command","flushes","faults"]
-
-def convert(s)
-  mul = case s[-1] 
-    when "b"
-      1
-    when "k"
-      1024
-    when "m"
-      1024 * 1024
-    when "g"
-      1024 * 1024 *1024
-    when "t"
-      1024 * 1024 * 1024 *1024
-    else 
-      raise "not convertible"
-    end
-  s[0..-1].to_f * mul
-end
-
+require 'mongodb-graphite'
+require 'optparse'
+require 'daemons'
+require 'simple-graphite'
 
+conf_dir = File.expand_path(ENV['MONGODB_GRAPHITE_DIR'] || '.')
 
+config = MongodbGraphite::Configuration.new(conf_dir)
 
+daemon_options = {
+  :multiple => false,
+  :monitor => true,
+  :backtrace => true,
+  :log_dir => config[:log_dir],
+  :log_output => true,
+  :dir_mode => :normal,
+  :dir => config[:pid_dir]
+}
 
+g_host, g_port = config[:graphite].split(':')
 
-
-
-
-
-def treat_replicaset(g,to_ignore,to_convert, output_average, output_sum ,prefix_common,host,port, shard_name)
-  #puts "shard : ", host,port
-  connHost = Mongo::Connection.new(host, port)
-  slaves = Array.new
-  connHost["admin"].command({ "isMaster" => 1 } )["hosts"].each { |slave| slaves << slave}
-  slaves.each { |slave|
-    Process.fork do
-    host, port = slave.split(':')
-    timestamp =  Time.now.getutc.to_i
-    result= `mongostat -n 60 --host #{host} --port #{port}   `.split("\n")
-    #timestamp +=  Time.now.getutc.to_i
-    #timestamp /= 2
-    headers, values = result[1], result[2..-1]
-    headers = headers.gsub(' %','').gsub('*','').gsub('idx miss','idxmiss').split
-    values= values.delete_if {|line| line.start_with?('insert')}.map { |line| line.gsub('*','').split}
-    values = values.transpose
-
-    prefix = prefix_common +shard_name+"."+ host +"."+port+"."
-    headers = headers.map { |key| prefix+key }
-    metrics = Hash[headers.zip(values)]
-    to_ignore.each { |key| metrics.delete(prefix + key) }
-    to_convert.each { |key| metrics[prefix + key] = metrics[prefix+key].map {|val| convert(val)}}
-    output_average.each { |key| metrics[prefix+key] = metrics[prefix+key].inject(0.0) {|sum,el| sum+(el.to_i) } / metrics[prefix+key].size }
-    output_sum.each { |key| metrics[prefix+key] = metrics[prefix+key].inject(0) {|sum,el| sum+(el.to_i)} }
-    g.log(timestamp, metrics)
-    end
-  }
+[:log_dir, :pid_dir].each do |d|
+  Dir.mkdir config[d] unless Dir.exist?(config[d])
 end
 
-hostt='localhost'
-portt=27017
-@conn = Mongo::Connection.new(hostt,portt)
-@isdbgrid  = {"isdbgrid" => 1}
-begin
-  @conn["admin"].command(@isdbgrid)["ok"] == 1
-  puts "mongos"
-  shards = Array.new
-  puts @conn["config"]["shards"].find().each { |shard|  shards << shard["host"] }
-  shards.each { |s|
-    puts s
-    Process.fork do
-    host,port =  s.split(/,|\//)[1].split(':')
-    treat_replicaset(g,to_ignore,to_convert, output_average, output_sum ,prefix_common,host,port, s.split(/,|\//)[0])
+Daemons.run_proc('mongodb-graphite', daemon_options) do
+  loop do
+    g = Graphite.new( :host => g_host, :port => g_port || '3333' )
+    @instance_list = MongodbGraphite::InstanceList.new(config)
+    @instance_list.each do |i|
+      begin
+        g.send_metrics(i.to_graphite)
+      rescue Exception => e
+        puts "Impossible to send metrics because of #{e}"
+        puts e.backtrace
+      end
     end
-  }
-rescue
-  puts "not mongos"
-  treat_replicaset(g,to_ignore,to_convert, output_average, output_sum ,prefix_common,hostt,portt, "direct")
+    sleep (config[:interval] || 5)
+  end
 end
-Process.wait

data/lib/mongodb-graphite.rb

@@ -0,0 +1,5 @@
+module MongodbGraphite
+  autoload :Configuration, 'mongodb-graphite/configuration'
+  autoload :InstanceList, 'mongodb-graphite/instance_list'
+  autoload :Instance, 'mongodb-graphite/instance'
+end

data/lib/mongodb-graphite/configuration.rb

@@ -0,0 +1,39 @@
+require 'yaml'
+
+module MongodbGraphite
+  class Configuration
+    def initialize(dir)
+      reload(dir)
+    end
+
+    def reload(dir)
+      @config = {
+        :log_dir => 'log',
+        :pid_dir => 'pids',
+        :graphite => 'localhost:3333',
+        :mongodb => 'localhost:27017',
+        :interval => 3, #in seconds
+        :instances_file => nil
+      }
+
+      config_file_path = File.join(dir, 'config', 'mongodb-graphite.yml')
+      if File.exist? config_file_path
+        config_file = YAML.load_file(config_file_path)
+        @config.keys.each do |key|
+          value = config_file[key.to_s]
+          @config[key] = value unless value.nil?
+        end
+      end
+
+      @config[:mongodb_graphite_dir] = dir
+
+      [:log_dir, :pid_dir, :instances_file].each do |k|
+        @config[k] = File.absolute_path(@config[k], dir)
+      end
+    end
+
+    def [](key)
+      @config[key]
+    end
+  end
+end

data/lib/mongodb-graphite/instance.rb

@@ -0,0 +1,101 @@
+require 'mongo'
+require 'simple-graphite'
+
+module Enumerable
+  def merge_all
+    self.inject({}) { |h1, h2| 
+      h1.merge! h2 
+    }
+  end
+end
+
+module MongodbGraphite
+  class Instance
+
+    OPTIONS = [:host, :port, :prefix_callback, :fields_to_ignore]
+
+    def initialize(name)
+      @name = name
+      @port = 27017
+      @host = 'localhost'
+      @fields_to_ignore = %w(host version process pid uptimeMillis localTime extra_info.note backgroundFlushing.last_finished repl.setName repl.hosts repl.arbiters repl.primary repl.me ok)
+      @prefix_callback = nil
+    end
+
+    def reload(options)
+      OPTIONS.each do |opt|
+        val = instance_variable_get "@#{opt}"
+        if options[opt] != val and not options[opt].nil?
+          instance_variable_set "@#{opt}", options.delete(opt)
+        end
+      end
+    end
+
+    OPTIONS.each do |opt|
+      define_method opt do
+        instance_variable_get "@#{opt}"
+      end
+    end
+
+    def to_s
+      "#{@name} #{@host}:#{@port}, #{@prefix_callback.class}"
+    end
+
+    def to_graphite
+      @stats = to_hash 
+      with_prefix = Hash.new
+      @stats.each do |k,v|
+        with_prefix[ [prefix, k].join('.')] = to_i(v)
+      end
+      with_prefix.reject { |k,v| ignored_fields.include? k }
+    end
+
+    private
+
+    def prefix
+      return @prefix_callback.call(@stats) unless @prefix_callback.nil?
+      nil
+    end
+
+    def to_i(v)
+      return v.to_i if v.respond_to?('to_i')
+      case v
+      when TrueClass
+        1
+      when FalseClass
+        0
+      else
+        nil
+      end
+    end
+
+    def ignored_fields
+      @fields_to_ignore.map { |f| [prefix,f].join('.') }
+    end
+
+    def connection
+      @connection ||= Mongo::MongoClient.new(host, port, :slave_ok => true, :connect_timeout => 5, :pool_timeout => 5)
+    end
+
+    def stats
+      connection['test'].command({ 'serverStatus' => 1})
+    end
+
+    def to_hash
+      s = stats
+      @process = s['process']
+      json_descent([], s).flatten.merge_all
+    end
+
+    def json_descent(pre, json)
+      json.map do |k,v|
+        key = pre + [k]
+        if v.is_a? BSON::OrderedHash
+          json_descent(key, v)
+        else
+          {key.join('.') => v }
+        end
+      end
+    end
+  end
+end

data/lib/mongodb-graphite/instance_list.rb

@@ -0,0 +1,46 @@
+module MongodbGraphite
+  class InstanceList
+    include Enumerable
+
+    attr_accessor :instances
+
+    def initialize(config)
+      @config = config
+      @instances = {}
+      reload
+    end
+
+    def reload
+      @instances = {}
+
+      file = @config[:instances_file]
+      if not file.nil?
+        #puts "Reading instance file from #{file}"
+        instance_eval(File.read(file),file)
+      end
+    end
+
+    def instance(name)
+      @options = {}
+      yield
+      @instances[name] ||= Instance.new(name)
+      @instances[name].reload(@options)
+      @options = nil
+    end
+
+    #dynamic definition of properties to have a nice dsl
+    Instance::OPTIONS.each do |opt|
+      define_method opt do |v|
+        @options[opt] = v
+      end
+    end
+
+
+    def each
+      @instances.each do |name,i|
+        yield i
+      end
+    end
+
+  end
+end

data/mongodb-graphite.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "mongodb-graphite"
-  s.version     = '0.1.5'
+  s.version     = '1.0.0'
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Grégoire Seux"]
   s.email       = ["g.seux@criteo.com"]
@@ -16,6 +16,6 @@ Gem::Specification.new do |s|
   #s.require_paths = ["lib"]
   
   s.add_dependency "mongo", ">= 1.5.2"
-  s.add_dependency "graphite", ">= 0.2.0"
-  s.add_dependency "peach", ">= 0.4"
+  s.add_dependency "simple-graphite", ">= 2.1.0"
+  s.add_dependency 'daemons'
 end

metadata

@@ -1,116 +1,95 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mongodb-graphite
-version: !ruby/object:Gem::Version 
-  hash: 17
-  prerelease: 
-  segments: 
-  - 0
-  - 1
-  - 5
-  version: 0.1.5
+version: !ruby/object:Gem::Version
+  version: 1.0.0
 platform: ruby
-authors: 
-- "Gr\xC3\xA9goire Seux"
+authors:
+- Grégoire Seux
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-03-15 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2013-06-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: mongo
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 1
-        - 5
-        - 2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
         version: 1.5.2
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: graphite
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 0
-        - 2
-        - 0
-        version: 0.2.0
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.2
+- !ruby/object:Gem::Dependency
+  name: simple-graphite
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.0
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: peach
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        - 4
-        version: "0.4"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id003
-description: ""
-email: 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
 - g.seux@criteo.com
-executables: 
+executables:
 - mongodb-graphite
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - Gemfile
 - Gemfile.lock
 - README.md
 - bin/mongodb-graphite
+- lib/mongodb-graphite.rb
+- lib/mongodb-graphite/configuration.rb
+- lib/mongodb-graphite/instance.rb
+- lib/mongodb-graphite/instance_list.rb
 - mongodb-graphite.gemspec
 homepage: https://github.com/kamaradclimber/mongodb-graphite
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
-summary: ""
+specification_version: 4
+summary: ''
 test_files: []
-