memcached_stats 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bb7d3b8523d38561f42308aca4abd8fda160fd71
+  data.tar.gz: f6bcbd334b14fee0a16c22a51d1b5056a65fd66f
+SHA512:
+  metadata.gz: d5ba210fc118497116689a2ee235f09d899a1903b8bc2c4e158c34908a892cce60a35d0d8bcab482fdecee0853aa919618eed0d6cf33c6014647d0a6e1c2bd1b
+  data.tar.gz: 2793035a28ceeab51f4c0a6400ab826d21208c18785f8edffa6496713b8d9701cee876f5ea2981c577092a704947192a016d6ffe911ebfcb456553bd5f105982

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 William Schneider
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.textile

@@ -0,0 +1,216 @@
+h1. Memcached Stats
+
+<i>*Note:* This gem is probably not yet ready for production environments. Use at your own discretion.</i>
+
+The goal of this gem is to provide overall memcached statistics (uptime, size, etc/) as well as an advanced look at cache slabs, providing slab item statistics as well as slab keys and values arranged hierarchically. As of the current release, this gem does not use the memcached binary protocol, therefore memcached versions prior to 1.4 (as well as 1.4+) should work fine with this gem. Please open an issue through github specifying your version of memcached if you find this is not the case.
+
+
+The following usage guidelines are subject to change...at least until 0.1.x release (which may or may not occur).
+
+h2. Usage
+
+h3. Basics
+
+<pre>
+<code>
+  $ gem install memcached_stats
+</code>
+</pre>
+
+(You may need to install the gem with @sudo@ prepended)
+
+The steps below assume usage from irb, however this gem can just as easily be used within a Rails project.
+
+First, require the @memcached_stats@ gem.
+
+<pre>
+<code>
+  require 'memcached_stats'
+</code>
+</pre>
+
+memcached_stats assumes a default host of @localhost@ and a port of @11211@. Values other than the default can be specified when creating a new @MemcachedStats@ object:
+
+<pre>
+<code>
+  stats = MemcachedStats.new(your_host, your_port)
+</code>
+</pre>
+
+After you create the new @MemcachedStats@ object, you can fetch the basic stats from memcached:
+
+<pre>
+<code>
+  stats.fetch_stats
+</code>
+</pre>
+
+h3. Summary Statistics
+
+Once you do this, you have access to the following default memcached statistics through the following instance accessor methods (via attributes courtesy of @method_missing@) which are described in greater detail "on the memcached github page":https://github.com/memcached/memcached/blob/master/doc/protocol.txt under the "General-purpose statistics" heading. The definitions below are copied directly from this document:
+
+*NOTE:* These memcached statistics methods are somewhat memcached version specific. They are defined in the text protocol of your particular memcached version, which can be viewed by visiting the github page linked above and switching to the tag which corresponds to your memcached version. Alternatively, via irb following the steps above, run @stats.summary.keys@ to view the keys which @MemcachedStats@ was able to attain.
+
+@pid@ - Process id of this server process
+@uptime@ - Number of secs since the server started
+@time@ - current UNIX time according to the server
+@version@ - Version string of this server 
+@pointer_size@ - Default size of pointers on the host OS (generally 32 or 64)
+@rusage_user@ - Accumulated user time for this process (seconds:microseconds)
+@rusage_system@ - Accumulated system time for this process (seconds:microseconds)
+@curr_items@ - Current number of items stored
+@total_items@ - Total number of items stored since the server started
+@bytes@ - Current number of bytes used to store items
+@curr_connections@ - Number of open connections
+@total_connections@ - Total number of connections opened since the server started running
+@connection_structures@ - Number of connection structures allocated by the server
+@cmd_get@ - Cumulative number of retrieval requests
+@cmd_set@ - Cumulative number of storage requests
+@cmd_flush@ - Cumulative number of flush requests
+@get_hits@ - Number of keys that have been requested and found present
+@get_misses@ - Number of items that have been requested and not found
+@delete_misses@ - Number of deletions requests for missing keys
+@delete_hits@ - Number of deletion requests resulting in an item being removed
+@incr_misses@ - Number of increment requests against missing keys
+@incr_hits@ - Number of successful increment requests
+@decr_misses@ - Number of decrement requests against missing keys
+@decr_hits@ - Number of successful decrement requests
+@cas_misses@ - Number of CAS reqs against missing keys
+@cas_hits@ - Number of successful CAS reqs
+@cas_badval@ - Number of CAS reqs for which a key was found, but the CAS value did not match
+@auth_cmds@ - Number of authentication commands handled, success or failure
+@auth_errors@ - Number of failed authentications
+@evictions@ - Number of valid items removed from cache to free memory for new items
+@reclaimed@ - Number of times an entry was stored using memory from an expired entry
+@bytes_read@ - Total number of bytes read by this server from network
+@bytes_written@ - Total number of bytes sent by this server to network
+@limit_maxbytes@ - Number of bytes this server is allowed to use for storage
+@accepting_conns@ - 
+@listen_disabled_num@ - 
+@threads@ - Number of worker threads requested
+@conn_yields@ - Number of times any connection yielded to another due to hitting the memcached @-R@ limit
+
+
+The following are defined as well, providing a few more useful metrics:
+
+@hit_ratio@ - Number of keys requested and found present over the total number of retrieval requests
+@miss_ratio@ - Number of keys requested and not found over the total number of retrieval requests
+@request_rate@ - Number of key retrieval requests over the memcached instance uptime (aka 'requests per second')
+@hit_rate@ - Number of keys requested and found present over the memcached instance uptime (aka 'cache hits per second')
+@miss_rate@ - Number of keys requested and not found over the memcached instance uptime (aka 'cache misses per second')
+
+
+
+h3. Slab Item Statistics
+
+The @fetch_stats@ instance method not only provides top level memcached statistics, but also gathers statistics for memcached slabs. Each slab is accessible via the @slabs@ accessor: a hash of hashes, the first of which is keyed by the slab_id, then the resultant hash is keyed by the statistic of interest. So for example:
+
+<pre>
+<code>
+  stats.slabs # => {"1"=>{"number"=>"1234", "age"=>"7000", "evicted"=>"0", "evicted_nonzero"=>"0", "evicted_time"=>"0", "outofmemory"=>"0", "tailrepairs"=>"0", "reclaimed"=>"0"}, "2"=>{"number"=>"122", "age"=>"161000", "evicted"=>"0", "evicted_nonzero"=>"0", "evicted_time"=>"0", "outofmemory"=>"0", "tailrepairs"=>"0", "reclaimed"=>"1"}...}
+  
+  stats.slabs["1"] # => {"number"=>"1234", "age"=>"7000", "evicted"=>"0", "evicted_nonzero"=>"0", "evicted_time"=>"0", "outofmemory"=>"0", "tailrepairs"=>"0", "reclaimed"=>"0"}
+  
+  stats.slabs["1"].number # => "1234"
+</code>
+</pre>
+
+
+"on the memcached github page":https://github.com/memcached/memcached/blob/master/doc/protocol.txt under the "Item statistics" heading, the slabs statistics represented in the slabs hash above are explained as follows:
+
+
+@number@ - Number of items presently stored in this class (expired items are not automatically excluded)
+@age@ - Age of the oldest item in the LRU
+@evicted@ - Number of times an item had to be evicted from the LRU before it expired
+@evicted_nonzero@ - Number of times an item which had an explicit expire time set had to be evicted from the LRU before it expired
+@evicted_time@ - Seconds since the last access for the most recent item evicted from this class (use this to judge how recently active your evicted data is)
+@outofmemory@ - Number of times the underlying slab class was unable to store a new item (this means you are running memcached with @-M@ or an eviction failed)
+@tailrepairs@ - Number of times we self-healed a slab with a refcount leak
+@reclaimed@ - Number of times an entry was stored using memory from an expired entry
+
+
+h3. Slab Keys
+
+
+One of the nice features of this gem is the ability to fetch all slab keys in a hierarchical hash. This is best explained with an example:
+
+Suppose your stored the following keys and corresponding values in memcached
+
+<pre>
+"User:bananastalktome:EmailCount" = 1
+"User:bananastalktome:Alerts" = 5
+"User:anotheruser:EmailCount" = 0
+"Post:Article:12:title" = "This article is cool"
+</pre>
+
+Using this gem, call @fetch_all_slab_keys@ on the @MemcachedStats@ object (@stats@) created above. The following hash will result, as an attribute called @slab_keys@:
+
+<pre>
+{
+  { 
+    "User" => 
+    { 
+      "bananastalktome" => 
+      { 
+        "EmailCount" => 
+        {
+        }
+      , "Alerts" =>
+        {
+        
+        }
+      }
+      , "anotheruser" =>
+      {
+        "EmailCount" =>
+        {
+        }
+      }      
+    }
+  , "Post" => 
+  { 
+    "Article" =>
+    { 
+      "12" =>
+      { 
+        "title" =>
+        {
+        }
+      }
+    }
+  }
+}
+</pre>
+
+Arranging the cache keys in this manner allows quick traversal down a hash to determine if a given key exists and, if it does, in which slab the key exists. For the lowest level of each tree in the hash, there is a @prod@ defined called @value@, which can be invoked to retrieve the value for that particular cache key. For example:
+
+<pre>
+<code>
+  stats.slab_keys["User"]["bananastalktome"]["EmailCount"].value # => 1
+</code>
+</pre>
+
+
+h2. Experimental Branch
+
+In the experimental branch of this project, I am messing with the ability to gather stats from multiple memcached instances configured in a cluster (as done in many production Rails projects) <i>in an intuitive way</i>. If you would like to help out with this, please 
+
+
+h2. Bugs and Feedback
+
+If you notice any bugs, would like any features added, or would just like to write code or tests ( <i>hint hint :)</i> ), please do not hesitate to do so! Either follow the *Note on Patches/Pull Requests* guidelines below, submit a issue through github, or email me directly at bananastalktome@gmail.com.
+
+
+h2. Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally. (I tend to be lax on tests, so if you want to make some for the code that I already have, you get bonus points!)
+* Commit, do not mess with Rakefile, VERSION, history, or other files not directly relevant to your contribution.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for fun request messages.
+
+h2. Copyright
+
+Copyright (c) 2010 William Schneider. See LICENSE for details.

data/lib/memcached_stats/base.rb

@@ -0,0 +1,150 @@
+class MemcachedStats
+  attr_reader :host, :port, :summary, :slabs, :slab_keys
+  
+  ## Altered code from watsonian/memcache_do gem
+  def self.service_connection(command, host, port, &block)
+    begin
+      data = ''
+      sock = TCPSocket.new(host, port)
+      sock.print("#{command}\r\n")
+      sock.flush
+      stats = sock.gets
+      while true
+        data += stats
+        break if ['END', 'OK', 'ERROR'].include? stats.strip
+        stats = sock.gets
+      end
+      sock.close
+      data
+    rescue => e
+      ## ToDo: REVISIT LATER, MAYBE RAISE SOMETHING
+      #puts e
+    end
+  end
+  
+  def initialize(host = "localhost", port = 11211)
+    @host = host
+    @port = port.to_i
+    @slab_keys = { }
+    self
+  end
+  
+  def fetch_stats
+    get_summary
+    get_slabs
+    self
+  end
+  
+  #def inspect
+    #[@host, @port, @summary]
+  #end
+
+#   # Commented out since I feel (at least at the moment) that it is out of the scope of what
+#   #   this gem should do.
+#   def flush
+#     MemcachedStats.service_connection("flush_all", @host, @port)
+#   end
+  
+  def fetch_all_slab_keys
+    get_slabs unless @slabs
+    @slabs.each do |slab|
+      fetch_single_slab_keys(slab)
+    end
+    true
+  end
+  
+  
+  def fetch_single_slab_keys(slab)
+    slab_id = slab.is_a?(Fixnum) ? slab : (slab.is_a?(Array) ? slab.first : nil)
+    
+    MemcachedStats.service_connection("stats cachedump #{slab_id} 0", @host, @port).sub("END\r\n","").each_line do |line|
+      #tmp = line.sub("ITEM ", "").split(" ").first.split(":")
+      
+      ## ToDo: Colon might not be delineator
+      
+      tmp = line.match(/ITEM\ (.*?)\ \[(\d*)\ b\;\ (\d*)\ s\]/).to_a[1..-1] # 0:key, 1:size, 2:expires
+      #ITEM User:3293:EmailCount [4 b; 1289245903 s]
+      #   [size; expires_timestamp]
+      
+      record_slab_data(tmp, slab_id)
+    end
+    true
+  end
+
+
+  protected
+  
+  def get_summary
+    @summary = Hash[*MemcachedStats.service_connection("stats", @host, @port).sub("END\r\n","").each_line.inject([]){|arr, line| arr.push(line.sub("STAT ", "").split(' '))}.flatten]
+    @summary["hit_ratio"] = @summary["get_hits"].to_f / @summary["cmd_get"].to_f
+    @summary["miss_ratio"] = @summary["get_misses"].to_f / @summary["cmd_get"].to_f
+    @summary["request_rate"] = @summary["cmd_get"].to_f / @summary["uptime"].to_i
+    @summary["hit_rate"] = @summary["get_hits"].to_f / @summary["uptime"].to_i
+    @summary["miss_rate"] = @summary["get_misses"].to_f / @summary["uptime"].to_i
+    @summary.freeze
+  end
+  
+  def get_slabs
+    @slabs = Hash.new{|h,k| h[k] = {} unless h.frozen? }
+    MemcachedStats.service_connection("stats items", @host, @port).sub("END\r\n","").each_line do |line|
+      l = (line.sub("STAT items:", "").split(' '))
+      k,v = l.first.split(":")
+      slabs[k][v] = l.last
+    end
+    @slabs.freeze
+  end
+  
+  private
+  
+  def record_slab_data(data, slab_id)
+    cache_key = data[0] #data.join(":")
+    key = cache_key.split(":")
+    first = key.shift
+    @slab_keys[first] = { } unless @slab_keys[first]
+    pointer = @slab_keys[first]
+    while elem = key.shift
+      pointer[elem] = { } unless pointer[elem]
+      pointer = pointer[elem]
+    end
+    
+    host = @host
+    port = @port
+    
+    pointer.define_singleton_method(:value) do
+      Proc.new do
+        #a = MemcachedStats.service_connection("get #{cache_key}", host, port).split
+        ## a => ["VALUE", "Billy:TestKey", "1", "29", "\x04\bI\"\bHey\x06:", "encoding\"", "US-ASCII", "END"]
+        #value = a[4..-4].join(" ")
+        #value += "\r#{a[-3]}\r#{a[-2]}"
+        
+        socket = TCPSocket.new(host, port)
+        socket.write("get #{cache_key}\r\n")
+        socket.gets
+        value = socket.read(data[1].to_i)
+        socket.close
+        
+        begin
+          Marshal.load value
+        rescue TypeError # Meaning marshalling failed
+          value
+        end
+      end
+    end
+    
+    ## ToDo: Add a delete key method??
+    
+    #pointer["slab"] = slab_id # Not really useful, I guess
+    #pointer["size"] = data[1] # The size (in bytes)
+    #pointer["expires"] = data[2] # Unix timestamp of expiration
+  end
+
+
+  def method_missing(method, *arguments, &block)
+    if @summary && @summary.key?(method.to_s)
+      @summary[method.to_s]
+    else
+      super
+    end
+  end
+
+end

data/lib/memcached_stats.rb

@@ -0,0 +1,6 @@
+require 'socket'
+
+
+Dir["#{File.dirname(__FILE__) + '/memcached_stats'}/**/*"].each do |filter|
+  require "#{filter}"
+end

data/test/helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'memcached_stats'
+
+class Test::Unit::TestCase
+end

data/test/test_memcached_stats.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestMemcachedStats < Test::Unit::TestCase
+  should "" do
+    
+  end
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: memcached_stats
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- William Schneider
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2010-12-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thoughtbot-shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Gather Memcached Stats as well as Slabs
+email: bananastalktome@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.textile
+files:
+- LICENSE
+- README.textile
+- lib/memcached_stats.rb
+- lib/memcached_stats/base.rb
+- test/helper.rb
+- test/test_memcached_stats.rb
+homepage: https://github.com/bananastalktome/memcached_stats
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--charset=UTF-8"
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 3
+summary: Gather Memcached Stats as well as Slabs
+test_files:
+- test/test_memcached_stats.rb
+- test/helper.rb