nutcracker-graphite 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2ac51b5f953ef068101cc9c5b2f4aac6b0370eb2
+  data.tar.gz: 7f2d673e8982941ae96c2d65bb92f89e6ea116db
+SHA512:
+  metadata.gz: 39344998867494f870f0c473533f13dbee63013ac3f8a0793ea9b4b8567390166603d4d411ab9a6d6dd2889ddded9b2c80ec13a997d65041e1d8d9883b9a93d1
+  data.tar.gz: aec3d483bbeaaeb8f8ac4514e7283a6ff33a1493d63c668e97fe93b38047ba4d200ccfd8def4b4f26403d256443f52465020d55ce853ddc9851e5a198c375718

data/README.md

@@ -0,0 +1,102 @@
+Nutcracker Graphite
+==================
+<a href="https://rubygems.org/gems/nutcracker-graphite"><img src=https://fury-badge.herokuapp.com/rb/nutcracker-graphite.png></a>
+
+[Nutcracker](https://github.com/kontera-technologies/nutcracker) plugin for sending cluster data to Graphite
+
+### DISCLAIMER
+this is still a work in progress...
+
+## Dependencies
+- ruby 1.9+
+- [Graphite](http://graphite.wikidot.com/)
+
+## Installation 
+Add this line to your application's Gemfile:
+```
+gem 'nutcracker-graphite'
+```
+
+And then execute:
+```
+$ bundle install
+```
+
+or just by
+```
+$ gem install nutcracker-graphite
+```
+
+## Usage
+example app
+```ruby
+require 'nutcracker'
+require 'nutcracker/graphite'
+
+# Start nutcracker
+nutcracker = Nutcracker.start(cluster: 'cluster.conf')
+
+# Start Graphite stats agent
+nutcracker.use(:graphite, graphite: 'graphite.example.com')
+# will send aggregated stats to graphite.example.com every 1 minute
+
+# Waiting....
+nutcracker.join
+```
+
+example output for two nodes cluster ( will be send every 1 minute )
+```
+nutcracker.<cluster-name>.<nutcracker-hostname>.client_eof 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.client_err 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.client_connections 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.server_ejects 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.forward_error 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.fragments 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_server_eof 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_server_err 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_server_timedout 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_server_connections 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_requests 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_request_bytes 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_responses 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_response_bytes 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_in_queue 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_in_queue_bytes 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_out_queue 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.cluster_out_queue_bytes 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.connections 95.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.used_memory 20173789840.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.used_memory_rss 20932427776.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.fragmentation 1.04 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.expired_keys 36732182.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.evicted_keys 12284.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.hits 1372163466.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.misses 58928722.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.keys 24347281.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.max_memory 21474836480.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node1>.hit_ratio 0.96 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_server_eof 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_server_err 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_server_timedout 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_server_connections 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_requests 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_request_bytes 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_responses 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_response_bytes 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_in_queue 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_in_queue_bytes 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_out_queue 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.cluster_out_queue_bytes 0.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.connections 95.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.used_memory 20173789840.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.used_memory_rss 20932427776.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.fragmentation 1.04 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.expired_keys 36732182.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.evicted_keys 12284.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.hits 1372163466.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.misses 58928722.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.keys 24347281.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.max_memory 21474836480.0 1369569180
+nutcracker.<cluster-name>.<nutcracker-hostname>.<node2>.hit_ratio 0.96 1369569180
+```
+

data/Rakefile

@@ -0,0 +1,18 @@
+$:.unshift File.expand_path '../lib', __FILE__
+require 'nutcracker/graphite'
+require 'rubygems/package_task'
+require 'rake/testtask'
+
+Nutcracker::GemSpec = eval File.read 'nutcracker-graphite.gemspec'
+
+Gem::PackageTask.new Nutcracker::GemSpec do |p|
+  p.gem_spec = Nutcracker::GemSpec
+end
+
+## Tests stuff
+task :default => :test
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push 'tests'
+  t.pattern = 'tests/**/*_test.rb'
+end

data/lib/nutcracker/graphite.rb

@@ -0,0 +1,98 @@
+require 'nutcracker/graphite/version'
+require 'graphite-api'
+require 'redis'
+
+module Nutcracker
+  module Graphite
+
+    def self.start nutcracker, options
+      Agent.new(nutcracker, options).start
+    end
+
+    class Agent
+      INTERVAL=60
+
+      attr_reader :nutcracker, :graphite
+
+      def initialize nutcracker, options
+        @nutcracker = nutcracker
+        @graphite = GraphiteAPI.new options.merge(:interval => INTERVAL)
+      end
+
+      def start
+        @task ||= graphite.every INTERVAL do |client|
+          client.metrics metrics parse nutcracker.stats
+        end
+        self
+      end
+
+      def stop
+        @task and @task.cancel
+      end
+
+      private
+
+      def metrics parsed_stats
+        data = parsed_stats.clone
+        escape = ->(s) { s.gsub(/\.|\:/,'_') }
+        hash = {}
+        data[:clusters].each do |cluster, cluster_data|
+          next unless ( nutcracker.config[cluster]["redis"] rescue false ) # skip memcached
+          cluster_key = ['nutcracker',cluster,data['source']].map(&escape).join('.')
+          cluster_data.each do |key, value|
+            hash[[cluster_key,key].join('.')] = value if value.is_a? Fixnum or value.is_a? Float
+          end
+
+          cluster_data[:nodes].each do |node, node_data|
+            node_key = "#{cluster_key}.#{escape.(node)}"
+            node_data.each do |key, value|
+              hash["#{node_key}.cluster_#{key}"] = value if value.is_a? Fixnum or value.is_a? Float
+            end
+            redis_info(node).each {|k,v| hash["#{node_key}.#{k}"] = v }
+          end
+
+        end
+        hash
+      end
+
+      def redis_info url
+        url = "redis://#{url}" unless url =~ /redis\:\/\//
+        redis = Redis.connect url: url
+        server_info = redis.info
+        db_size     = redis.dbsize
+        max_memory  = redis.config(:get, 'maxmemory')['maxmemory'].to_i
+        redis.quit
+        {
+          'connections'     => server_info['connected_clients'].to_i,
+          'used_memory'     => server_info['used_memory'].to_f,
+          'used_memory_rss' => server_info['used_memory_rss'].to_f,
+          'fragmentation'   => server_info['mem_fragmentation_ratio'].to_f,
+          'expired_keys'    => server_info['expired_keys'].to_i,
+          'evicted_keys'    => server_info['evicted_keys'].to_i,
+          'hits'            => server_info['keyspace_hits'].to_i,
+          'misses'          => server_info['keyspace_misses'].to_i,
+          'keys'            => db_size,
+          'max_memory'      => max_memory,
+          'hit_ratio'       => 0
+        }.tap {|d| d['hit_ratio'] = d['hits'].to_f / (d['hits']+d['misses']).to_f if d['hits'] > 0 }
+      end
+
+
+      def parse stats
+        { :clusters => {} }.tap do |data|
+          (stats.dup).each do |key, value|
+            (data[key] = value and next) if !value.is_a? Hash
+            data[:clusters][key] = value
+            data[:clusters][key][:nodes] = {}
+            data[:clusters][key].each do |key2,value2|
+              if value2.kind_of? Hash and key2.is_a? String
+                data[:clusters][key][:nodes][key2] = data[:clusters][key].delete(key2)
+              end
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/nutcracker/graphite/version.rb

@@ -0,0 +1,5 @@
+module Nutcracker
+  module Graphite
+    VERSION="0.0.1"
+  end
+end

data/tests/fixtures/expected_metrics_wo_redis_info.ruby_hash

@@ -0,0 +1 @@
+{"nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.client_eof"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.client_err"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.client_connections"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.server_ejects"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.forward_error"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.fragments"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_server_eof"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_server_err"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_server_timedout"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_server_connections"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_requests"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_request_bytes"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_responses"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_response_bytes"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_in_queue"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_in_queue_bytes"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_out_queue"=>0, "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.cluster_out_queue_bytes"=>0}

data/tests/fixtures/expected_redis_info.json

@@ -0,0 +1 @@
+{"connections":96,"used_memory":20042110520.0,"used_memory_rss":20905414656.0,"fragmentation":1.04,"expired_keys":36705179,"evicted_keys":12284,"hits":1370531418,"misses":58886159,"keys":24296159,"max_memory":21474836480,"hit_ratio":0.9588040891986317}

data/tests/fixtures/expected_stats.ruby_hash

@@ -0,0 +1 @@
+{:clusters=>{"page_data_cluster"=>{"client_eof"=>0, "client_err"=>0, "client_connections"=>0, "server_ejects"=>0, "forward_error"=>0, "fragments"=>0, :nodes=>{"node1:6379"=>{"server_eof"=>0, "server_err"=>0, "server_timedout"=>0, "server_connections"=>0, "requests"=>0, "request_bytes"=>0, "responses"=>0, "response_bytes"=>0, "in_queue"=>0, "in_queue_bytes"=>0, "out_queue"=>0, "out_queue_bytes"=>0}}}}, "service"=>"nutcracker", "source"=>"Eran-Levis-MacBook-Pro.local", "version"=>"0.2.4", "uptime"=>53, "timestamp"=>1369558826}

data/tests/fixtures/redis_info.json

@@ -0,0 +1 @@
+{"info":{"redis_version":"2.4.10","redis_git_sha1":"00000000","redis_git_dirty":"0","arch_bits":"64","multiplexing_api":"epoll","gcc_version":"4.1.2","process_id":"15264","uptime_in_seconds":"5247213","uptime_in_days":"60","lru_clock":"641250","used_cpu_sys":"40451.50","used_cpu_user":"172259.69","used_cpu_sys_children":"0.00","used_cpu_user_children":"0.00","connected_clients":"96","connected_slaves":"0","client_longest_output_list":"0","client_biggest_input_buf":"0","blocked_clients":"0","used_memory":"20042110520","used_memory_human":"18.67G","used_memory_rss":"20905414656","used_memory_peak":"21474896448","used_memory_peak_human":"20.00G","mem_fragmentation_ratio":"1.04","mem_allocator":"jemalloc-2.2.5","loading":"0","aof_enabled":"0","changes_since_last_save":"8316280672","bgsave_in_progress":"0","last_save_time":"1364314088","bgrewriteaof_in_progress":"0","total_connections_received":"64618","total_commands_processed":"18247026966","expired_keys":"36705179","evicted_keys":"12284","keyspace_hits":"1370531418","keyspace_misses":"58886159","pubsub_channels":"0","pubsub_patterns":"0","latest_fork_usec":"0","vm_enabled":"0","role":"master","db0":"keys=24296116,expires=739455","db2":"keys=14178,expires=0"},"dbsize":24296159,"memory":{"maxmemory":"21474836480"}}

data/tests/fixtures/stats.json

@@ -0,0 +1 @@
+{"service":"nutcracker","source":"Eran-Levis-MacBook-Pro.local","version":"0.2.4","uptime":53,"timestamp":1369558826,"page_data_cluster":{"client_eof":0,"client_err":0,"client_connections":0,"server_ejects":0,"forward_error":0,"fragments":0,"node1:6379":{"server_eof":0,"server_err":0,"server_timedout":0,"server_connections":0,"requests":0,"request_bytes":0,"responses":0,"response_bytes":0,"in_queue":0,"in_queue_bytes":0,"out_queue":0,"out_queue_bytes":0}}}

data/tests/minitest_helper.rb

@@ -0,0 +1,31 @@
+$:.unshift File.expand_path '../../lib', __FILE__
+gem 'minitest'
+
+require 'minitest/autorun'
+require 'mocha/setup'
+require 'nutcracker/graphite'
+require 'tempfile'
+require 'fileutils'
+require 'json'
+
+module Nutcracker
+  module Graphite
+    class Test < ::Minitest::Test
+      def fixture name
+        File.expand_path("../fixtures/#{name}", __FILE__)
+      end
+
+      def load_fixture name
+        raw = File.read fixture name
+        if name.end_with? '.json'
+          JSON.parse raw
+        elsif name.end_with? '.ruby_hash'
+          eval raw
+        else
+          raw
+        end
+      end
+
+    end
+  end
+end

data/tests/unit/agent_test.rb

@@ -0,0 +1,51 @@
+require_relative '../minitest_helper'
+
+module Nutcracker
+  module Graphite
+    class AgentTester < Test
+
+      def setup
+        @nutcracker = mock
+        @options = {:graphite => 'shuki'}
+        @agent = Agent.new( @nutcracker, @options )
+      end
+
+      attr_reader :agent, :options, :nutcracker
+
+      def test_stats_parser
+        assert_equal load_fixture('expected_stats.ruby_hash'),
+          agent.send(:parse,load_fixture('stats.json'))
+      end
+
+      def test_metrics_generator
+        nutcracker.expects(:config).returns({'page_data_cluster' => {'redis' => true}})
+        agent.expects(:redis_info).with('node1:6379').returns({})
+
+        assert_equal load_fixture('expected_metrics_wo_redis_info.ruby_hash') ,
+          agent.send(:metrics,load_fixture('expected_stats.ruby_hash'))
+
+        nutcracker.expects(:config).returns({'page_data_cluster' => {'redis' => true}})
+        agent.expects(:redis_info).with('node1:6379').returns({'blabla' => 20})
+
+        expected = load_fixture('expected_metrics_wo_redis_info.ruby_hash').merge(
+          "nutcracker.page_data_cluster.Eran-Levis-MacBook-Pro_local.node1_6379.blabla"=>20
+        )
+
+        assert_equal expected, agent.send(:metrics,load_fixture('expected_stats.ruby_hash'))
+      end
+
+      def test_redis_info
+        data = load_fixture 'redis_info.json'
+        redis = mock
+        Redis.expects(:connect).with(url: 'redis://node1:6379').returns(redis)
+        redis.expects(:info).returns(data['info'])
+        redis.expects(:dbsize).returns(data['dbsize'])
+        redis.expects('config').with(:get,'maxmemory').returns(data['memory'])
+        redis.expects(:quit)
+        assert_equal load_fixture('expected_redis_info.json'),
+          agent.send(:redis_info,'node1:6379')
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: nutcracker-graphite
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Eran Barak Levi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: graphite-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: nutcracker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.4.beta3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.4.beta3
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+description: Nutcracker plugin for sending cluster statistics to Graphite
+email: eran@kontera.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/nutcracker/graphite/version.rb
+- lib/nutcracker/graphite.rb
+- tests/fixtures/expected_metrics_wo_redis_info.ruby_hash
+- tests/fixtures/expected_redis_info.json
+- tests/fixtures/expected_stats.ruby_hash
+- tests/fixtures/redis_info.json
+- tests/fixtures/stats.json
+- tests/minitest_helper.rb
+- tests/unit/agent_test.rb
+homepage: http://www.kontera.com
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.5
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: ruby-nutcracker-graphite
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Send Twemproxy/nutcracker statistics to Graphite
+test_files: []