remcached 0.3.1

data/.gitignore

@@ -0,0 +1,2 @@
+*#
+*~

data/README.rst

@@ -0,0 +1,105 @@
+remcached
+=========
+
+* **Ruby EventMachine memCACHED client implementation**
+* provides a direct interface to the memcached protocol and its
+  semantics
+* uses the memcached `binary protocol`_ to reduce parsing overhead on
+  the server side (requires memcached >= 1.3)
+* supports multiple servers with simple round-robin key hashing
+  (**TODO:** implement the libketama algorithm) in a fault-tolerant
+  way
+* writing your own abstraction layer is recommended
+* uses RSpec
+* partially documented in RDoc-style
+
+
+Callbacks
+---------
+
+Each request `may` be passed a callback. These are not two-cased
+(success & failure) EM deferrables, but standard Ruby callbacks. The
+rationale behind this is that there are no usual success/failure
+responses, but you will want to evaluate a ``response[:status]``
+yourself to check for cache miss, version conflict, network
+disconnects etc.
+
+A callback may be kept if it returns ``:proceed`` to catch
+multi-response commands such as ``STAT``.
+
+remcached has been built with **fault tolerance** in mind: a callback
+will be called with just ``{:status => Memcached::Errors::DISCONNECTED}``
+if the network connection has went away. Thus, you can expect your
+callback will be called, except of course you're using `quiet`
+commands. In that case, only a "non-usual response" from the server or
+a network failure will invoke your block.
+
+
+Multi commands
+--------------
+
+The technique is described in the `binary protocol`_ spec in section
+**4.2**. ``Memcached.multi_operation`` will help you exactly with
+that, sending lots of those `quiet` commands, except for the last,
+which will be a `normal` command to trigger an acknowledge for all
+commands.
+
+This is of course implemented per-server to accomodate
+load-balancing.
+
+
+Usage
+-----
+
+First, pass your memcached servers to the library::
+
+    Memcached.servers = %w(localhost localhost:11212 localhost:11213)
+
+Note that it won't be connected immediately. Use ``Memcached.usable?``
+to check. This however complicates your own code and you can check
+``response[:status] == Memcached::Errors::DISCONNECTED`` for network
+errors in all your response callbacks.
+
+Further usage is pretty straight-forward::
+
+    Memcached.get(:key => 'Hello') do |response|
+      case response[:status]
+        when Memcached::Errors::NO_ERROR
+          use_cached_value response[:value] # ...
+        when Memcached::Errors::KEY_NOT_FOUND
+          refresh_cache! # ...
+        when Memcached::Errors::DISCONNECTED
+          proceed_uncached # ...
+        else
+          cry_for_help # ...
+        end
+      end
+    end
+    Memcached.set(:key => 'Hello', :value => 'World',
+                  :expiration => 600) do |response|
+      case response[:status]
+        when Memcached::Errors::NO_ERROR
+          # That's good
+        when Memcached::Errors::DISCONNECTED
+	  # Maybe stop filling the cache for now?
+        else
+          # What could've gone wrong?
+        end
+      end
+    end
+
+Multi-commands may require a bit of precaution::
+
+    Memcached.multi_get([{:key => 'foo'},
+                         {:key => 'bar'}]) do |responses|
+      # responses is now a hash of Key => Response
+    end
+
+It's not guaranteed that any of these keys will be present in the
+response. Moreover, they may be present even if they are a usual
+response because the last request is always non-quiet.
+
+
+**HAPPY CACHING!**
+
+.. _binary protocol: http://code.google.com/p/memcached/wiki/MemcacheBinaryProtocol

data/Rakefile

@@ -0,0 +1,13 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "remcached"
+    gemspec.summary = "Ruby EventMachine memcached client"
+    gemspec.description = gemspec.summary
+    gemspec.email = "astro@spaceboyz.net"
+    gemspec.homepage = "http://github.com/astro/remcached/"
+    gemspec.authors = ["Stephan Maka"]
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:minor: 3
+:patch: 1
+:major: 0

data/examples/fill.rb

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+
+# Experimentally determine how many items fit in your memcached
+# instance. Adjust parameters below for your scenario.
+
+BATCH_SIZE = 1000
+KEY_SIZE = 20
+VALUE_SIZE = 20
+
+
+$: << File.dirname(__FILE__) + "/../lib"
+require 'remcached'
+
+EM.run do
+  @total = 0
+
+  # Action
+  def fill
+    old_total = @total
+
+    reqs = (1..BATCH_SIZE).map {
+      @total += 1
+      { :key => sprintf("%0#{KEY_SIZE}X", @total),
+        :value => sprintf("%0#{VALUE_SIZE}X", @total)
+      }
+    }
+    Memcached.multi_add(reqs) do |resps|
+      resps.each do |key,resp|
+        case resp[:status]
+        when Memcached::Errors::NO_ERROR
+          :ok
+        when Memcached::Errors::KEY_EXISTS
+          @total -= 1
+        else
+          puts "Cannot set #{key}: status=#{resp[:status].inspect}"
+          @total -= 1
+        end
+      end
+
+      puts "Added #{@total - old_total}, now: #{@total}"
+      if Memcached.usable?
+        stats = {}
+        Memcached.usable_clients[0].stats do |resp|
+          if resp[:key] != ''
+            stats[resp[:key]] = resp[:value]
+          else
+            puts "Stats: #{stats['bytes']} bytes in #{stats['curr_items']} of #{stats['total_items']} items"
+          end
+        end
+
+        # Next round:
+        fill
+      else
+        EM.stop
+      end
+    end
+  end
+
+  # Initialization & start
+  Memcached.servers = %w(localhost)
+  @t = EM::PeriodicTimer.new(0.01) do
+    if Memcached.usable?
+      puts "Connected to server"
+      @t.cancel
+      fill
+    else
+      puts "Waiting for server connection..."
+    end
+  end
+end

data/lib/remcached/client.rb

@@ -0,0 +1,143 @@
+require 'eventmachine'
+
+module Memcached
+  class Connection < EventMachine::Connection
+    def self.connect(host, port=11211, &connect_callback)
+      df = EventMachine::DefaultDeferrable.new
+      df.callback &connect_callback
+
+      EventMachine.connect(host, port, self) do |me|
+        me.instance_eval { 
+          @host, @port = host, port
+          @connect_deferrable = df
+        }
+      end
+    end
+
+    def connected?
+      @connected
+    end
+
+    def reconnect
+      @connect_deferrable = EventMachine::DefaultDeferrable.new
+      super @host, @port
+      @connect_deferrable
+    end
+
+    def post_init
+      @recv_buf = ""
+      @recv_state = :header
+      @connected = false
+    end
+
+    def connection_completed
+      @connected = true
+      @connect_deferrable.succeed(self)
+    end
+
+    RECONNECT_DELAY = 10
+    RECONNECT_JITTER = 5
+    def unbind
+      @connected = false
+      EventMachine::Timer.new(RECONNECT_DELAY + rand(RECONNECT_JITTER),
+                              method(:reconnect))
+    end
+
+    def send_packet(pkt)
+      send_data pkt.to_s
+    end
+
+    def receive_data(data)
+      @recv_buf += data
+
+      done = false
+      while not done
+
+        if @recv_state == :header && @recv_buf.length >= 24
+          @received = Response.parse_header(@recv_buf[0..23])
+          @recv_buf = @recv_buf[24..-1]
+          @recv_state = :body
+
+        elsif @recv_state == :body && @recv_buf.length >= @received[:total_body_length]
+          @recv_buf = @received.parse_body(@recv_buf)
+          receive_packet(@received)
+
+          @recv_state = :header
+
+        else
+          done = true
+        end
+      end
+    end
+  end
+
+  class Client < Connection
+    def post_init
+      super
+      @opaque_counter = 0
+      @pending = []
+    end
+
+    def unbind
+      super
+      @pending.each do |opaque, callback|
+        callback.call :status => Errors::DISCONNECTED
+      end
+      @pending = []
+    end
+
+    def send_request(pkt, &callback)
+      @opaque_counter += 1
+      @opaque_counter %= 1 << 32
+      pkt[:opaque] = @opaque_counter
+      send_packet pkt
+
+      if callback
+        @pending << [@opaque_counter, callback]
+      end
+    end
+
+    ##
+    # memcached responses possess the same order as their
+    # corresponding requests. Therefore quiet requests that have not
+    # yielded responses will be dropped silently to free memory from
+    # +@pending+
+    #
+    # When a callback has been fired and returned +:proceed+ without a
+    # succeeding packet, we still keep it referenced around for
+    # commands such as STAT which has multiple response packets.
+    def receive_packet(response)
+      pending_pos = nil
+      pending_callback = nil
+      @pending.each_with_index do |(pending_opaque,pending_cb),i|
+        if response[:opaque] == pending_opaque
+          pending_pos = i
+          pending_callback = pending_cb
+          break
+        end
+      end
+
+      if pending_pos
+        @pending = @pending[pending_pos..-1]
+        begin
+          if pending_callback.call(response) != :proceed
+            @pending.shift
+          end
+        rescue Exception => e
+          $stderr.puts "#{e.class}: #{e}\n" + e.backtrace.join("\n")
+        end
+      end
+    end
+
+    # Callback will be called multiple times
+    def stats(contents={}, &callback)
+      send_request Request::Stats.new(contents) do |result|
+        callback.call result
+
+        if result[:status] == Errors::NO_ERROR && result[:key] != ''
+          :proceed
+        end
+      end
+    end
+  end
+end

data/lib/remcached/const.rb

@@ -0,0 +1,64 @@
+module Memcached
+  module Datatypes
+    RAW_BYTES = 0x00
+  end
+
+  module Errors
+    NO_ERROR = 0x0000
+    KEY_NOT_FOUND = 0x0001
+    KEY_EXISTS = 0x0002
+    VALUE_TOO_LARGE = 0x0003
+    INVALID_ARGS = 0x0004
+    ITEM_NOT_STORED = 0x0005
+    NON_NUMERIC_VALUE = 0x0006
+
+    DISCONNECTED = 0xffff
+  end
+
+  module Commands
+    GET = 0x00
+    SET = 0x01
+    ADD = 0x02
+    REPLACE = 0x03
+    DELETE = 0x04
+    INCREMENT = 0x05
+    DECREMENT = 0x06
+    QUIT = 0x07
+    STAT = 0x10
+    GETQ = 0x09
+    SETQ = 0x11
+    ADDQ = 0x12
+    DELETEQ = 0x14
+
+=begin
+   Possible values of the one-byte field:
+   0x00    Get
+   0x01    Set
+   0x02    Add
+   0x03    Replace
+   0x04    Delete
+   0x05    Increment
+   0x06    Decrement
+   0x07    Quit
+   0x08    Flush
+   0x09    GetQ
+   0x0A    No-op
+   0x0B    Version
+   0x0C    GetK
+   0x0D    GetKQ
+   0x0E    Append
+   0x0F    Prepend
+   0x10    Stat
+   0x11    SetQ
+   0x12    AddQ
+   0x13    ReplaceQ
+   0x14    DeleteQ
+   0x15    IncrementQ
+   0x16    DecrementQ
+   0x17    QuitQ
+   0x18    FlushQ
+   0x19    AppendQ
+   0x1A    PrependQ
+=end
+  end
+end

data/lib/remcached/pack_array.rb

@@ -0,0 +1,46 @@
+##
+# Works exactly like Array#pack and String#unpack, except that it
+# inverts 'q' & 'Q' prior packing/after unpacking. This is done to
+# achieve network byte order for these values on a little-endian machine.
+#
+# FIXME: implement check for big-endian machines.
+module Memcached::PackArray
+  def self.pack(ary, fmt1)
+    fmt2 = ''
+    values = []
+    fmt1.each_char do |c|
+      if c == 'Q' || c == 'q'
+        fmt2 += 'a8'
+        values << [ary.shift].pack(c).reverse
+      else
+        fmt2 += c
+        values << ary.shift
+      end
+    end
+
+    values.pack(fmt2)
+  end
+
+  def self.unpack(buf, fmt1)
+    fmt2 = ''
+    reverse = []
+    i = 0
+    fmt1.each_char do |c|
+      if c == 'Q' || c == 'q'
+        fmt2 += 'a8'
+        reverse << [i, c]
+      else
+        fmt2 += c
+      end
+      i += 1
+    end
+
+    ary = buf.unpack(fmt2)
+
+    reverse.each do |i, c|
+      ary[i], = ary[i].reverse.unpack(c)
+    end
+
+    ary
+  end
+end