remix-stash 0.9.0 → 0.9.6

data/.gitignore

@@ -0,0 +1 @@
+pkg

data/LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2009 Paperless Post Inc.
+Copyright (c) 2009 Brian Mitchell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,32 @@
+Real docs coming soon! Check out the examples directory for more.
+
+# Quick Specs
+
+New API! I've rethought a lot of the API and this comes with a lot of new capabilities. More work is being done on making it as expressive as possible without terrible overhead. This includes vectorized keys which allow emulation of partial cache clearing as well as nice shortcuts like eval and gate for expressions. Options, clusters, and implicit scope are easy to manage on a stash-by-stash basis. Keys are also easy to pass in as it will create composite keys from whatever you pass in (as long as it has to_s) so no more ugly string interpolation all over the place.
+
+It's fast (faster than memcache-client). It's simple (pure ruby and only a few hundred lines). It's tested (shoulda). Of course, because it's pure ruby it will run almost anywhere as well unlike many other clients.
+
+It does require memcached 1.4+ but you should be running that anyway (if you aren't, upgrade already).
+
+Take a look and let me know what you think!
+
+# TODO
+
+* namespacing
+* implement the rest of the memcached 1.4 binary API (replace, append, prepend)
+* allow swappable cluster types for consistent hashing, ketama, etc...
+* failsafe marshal load
+* support non-marshal value dumps configured per stash
+* support multi vector sets
+* thread safe cluster
+* add block form
+* quiet/multi command forms (will require a protocol refactoring most likely)
+* server pings
+* complete stats API
+* incr/decr should take default value flags
+* get/set add/replace read/write should allow a CAS flag to be passed
+* accelerated binary API implementation with Ruby fallback
+* redis support for vectors and/or value
+* large key handling support
+* UDP support (more experimentation on the tradeoffs)
+* EventMachine integration (non-blocking?)

data/Rakefile

@@ -0,0 +1,12 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = 'remix-stash'
+    gemspec.summary = 'Remix your memcache'
+    gemspec.email = 'binary42@gmail.com'
+    gemspec.homepage = 'http://github.com/binary42/remix-stash'
+    gemspec.authors = ['Brian Mitchell']
+  end
+rescue LoadError
+  puts 'Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com'
+end

data/VERSION

@@ -0,0 +1 @@
+0.9.6

data/benchmarks/get_set.rb

@@ -0,0 +1,35 @@
+require 'benchmark'
+require File.dirname(__FILE__) + '/../harness'
+
+LARGE_NUMBER = 50_000
+
+Benchmark.bmbm do |b|
+  b.report('get/set remix-stash') do
+    LARGE_NUMBER.times {|n|
+      stash[:abcxyz123] = n
+      stash[:abcxyz123]
+    }
+  end
+  b.report('get/set remix-stash named') do
+    LARGE_NUMBER.times {|n|
+      stash(:stuff)[:abcxyz123] = n
+      stash(:stuff)[:abcxyz123]
+    }
+  end
+  if defined?(CCache)
+    b.report('get/set memcached') do
+      LARGE_NUMBER.times {|n|
+        CCache.set('abcxyz123', n, 0 , true)
+        CCache.get('abcxyz123', true)
+      }
+    end
+  end
+  if defined?(RCache)
+    b.report('get/set memcache-client') do
+      LARGE_NUMBER.times {|n|
+        RCache.set('abcxyz123', n)
+        RCache.get('abcxyz123')
+      }
+    end
+  end
+end

data/benchmarks/payload.rb

@@ -0,0 +1,125 @@
+require 'benchmark'
+require File.dirname(__FILE__) + '/../harness'
+
+LARGE_NUMBER = 20_000
+
+huge_value = 'a' * 100_000
+large_value = 'b' * 20_000
+med_value = 'c' * 2_000
+small_value = 'd' * 100
+tiny_value = 'e'
+
+KEY = 'abc123xyz'
+
+Benchmark.bmbm do |b|
+  b.report('100k remix-stash') do
+    LARGE_NUMBER.times {
+      stash.write(KEY, huge_value)
+      stash.read(KEY)
+    }
+  end
+  if defined?(CCache)
+    b.report('100k memcached') do
+      LARGE_NUMBER.times {
+        CCache.set(KEY, huge_value, 0, false)
+        CCache.get(KEY, false)
+      }
+    end
+  end
+  if defined?(RCache)
+    b.report('100k memcache-client') do
+      LARGE_NUMBER.times {
+        RCache.set(KEY, huge_value, 0, true)
+        RCache.get(KEY, true)
+      }
+    end
+  end
+  b.report('20k remix-stash') do
+    LARGE_NUMBER.times {
+      stash.write(KEY, large_value)
+      stash.read(KEY)
+    }
+  end
+  if defined?(CCache)
+    b.report('20k memcached') do
+      LARGE_NUMBER.times {
+        CCache.set(KEY, large_value, 0, false)
+        CCache.get(KEY, false)
+      }
+    end
+  end
+  if defined?(RCache)
+    b.report('20k memcache-client') do
+      LARGE_NUMBER.times {
+        RCache.set(KEY, large_value, 0, true)
+        RCache.get(KEY, true)
+      }
+    end
+  end
+  b.report('2k remix-stash') do
+    LARGE_NUMBER.times {
+      stash.write(KEY, med_value)
+      stash.read(KEY)
+    }
+  end
+  if defined?(CCache)
+    b.report('2k memcached') do
+      LARGE_NUMBER.times {
+        CCache.set(KEY, med_value, 0, false)
+        CCache.get(KEY, false)
+      }
+    end
+  end
+  if defined?(RCache)
+    b.report('2k memcache-client') do
+      LARGE_NUMBER.times {
+        RCache.set(KEY, med_value, 0, true)
+        RCache.get(KEY, true)
+      }
+    end
+  end
+  b.report('100b remix-stash') do
+    LARGE_NUMBER.times {
+      stash.write(KEY, small_value)
+      stash.read(KEY)
+    }
+  end
+  if defined?(CCache)
+    b.report('100b memcached') do
+      LARGE_NUMBER.times {
+        CCache.set(KEY, small_value, 0, false)
+        CCache.get(KEY, false)
+      }
+    end
+  end
+  if defined?(RCache)
+    b.report('100b memcache-client') do
+      LARGE_NUMBER.times {
+        RCache.set(KEY, small_value, 0, true)
+        RCache.get(KEY, true)
+      }
+    end
+  end
+  b.report('1b remix-stash') do
+    LARGE_NUMBER.times {
+      stash.write(KEY, tiny_value)
+      stash.read(KEY)
+    }
+  end
+  if defined?(CCache)
+    b.report('1b memcached') do
+      LARGE_NUMBER.times {
+        CCache.set(KEY, tiny_value, 0, false)
+        CCache.get(KEY, false)
+      }
+    end
+  end
+  if defined?(RCache)
+    b.report('1b memcache-client') do
+      LARGE_NUMBER.times {
+        RCache.set(KEY, tiny_value, 0, true)
+        RCache.get(KEY, true)
+      }
+    end
+  end
+end

data/examples/eval.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '/../harness'
+
+stuff = stash(:stuff)
+
+stuff.eval(:ans) {42}
+p stuff.get(:ans)
+
+stuff.eval(:ans) {fail 'Cache miss'}

data/examples/gate.rb

@@ -0,0 +1,6 @@
+require File.dirname(__FILE__) + '/../harness'
+
+p stash.gate(:x) {p :miss}
+p :set_x
+stash[:x] = true
+p stash.gate(:x) {p :miss}

data/examples/getset.rb

@@ -0,0 +1,13 @@
+require File.dirname(__FILE__) + '/../harness'
+
+# Get and set a simple key
+stash.set('answer', 42)
+p stash.get('answer')
+
+# Alternate methods
+stash[:answer] = :fortytwo
+p stash[:answer]
+
+# Composite keys
+stash[1,2,3,4] = 5
+p stash[1,2,3,4]

data/examples/scope.rb

@@ -0,0 +1,17 @@
+require File.dirname(__FILE__) + '/../harness'
+
+a = 0
+
+stash(:x).scope {a}
+
+stash(:x)[1] = 2
+
+p stash(:x)[1]
+
+a += 1
+
+p stash(:x)[1]
+
+a = 0
+
+p stash(:x)[1]

data/examples/stash.rb

@@ -0,0 +1,16 @@
+require File.dirname(__FILE__) + '/../harness'
+
+one = stash(:one)
+two = stash(:two)
+
+one[:x] = 10
+two[:x] = 12
+
+p one[:x]
+p two[:x]
+
+p :clearing
+one.clear
+
+p one[:x]
+p two[:x]

data/harness.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/lib'
+require 'remix/stash'
+
+begin
+  require 'memcached'
+  CCache = Memcached.new('localhost:11211')
+rescue
+  puts "memcached not found (skipping)"
+end
+
+begin
+  require 'memcache'
+  RCache = MemCache.new('localhost:11211')
+rescue
+  puts "memcached-client not found (skipping)"
+end
+
+stash.clear

data/lib/remix/stash/cluster.rb

@@ -0,0 +1,79 @@
+require 'socket'
+require 'digest/md5'
+
+class Remix::Stash::Cluster
+  include Socket::Constants
+
+  @@connections = {}
+
+  attr_reader :hosts
+
+  def initialize(hosts)
+    @hosts = hosts.map {|x|
+      host, port = x.split(':')
+      [x, host, (port || 11211).to_i]
+    }.sort_by {|(_,h,p)| [h,p]}
+  end
+
+  def each
+    @hosts.each do |h|
+      begin
+        io = host_to_io(*h)
+        break yield(io)
+      rescue Errno::EPIPE, Errno::ECONNRESET, Remix::Stash::ProtocolError
+        io.close
+        retry
+      rescue Errno::EAGAIN, Errno::ECONNREFUSED
+        next
+      end
+    end
+  end
+
+  # Note: Later, I'd like to support richer cluster definitions.
+  # This should do the trick for now... and it's fast.
+  def select(key)
+    count = @hosts.size
+    hash = nil
+    if count > 1
+      digest = Digest::MD5.digest(key)
+      hash = digest.unpack("L")[0]
+    else
+      hash = 0
+    end
+    count.times do |try|
+      begin
+        io = host_to_io(*@hosts[(hash + try) % count])
+        return yield(io)
+      rescue Errno::EPIPE, Errno::ECONNRESET, Remix::Stash::ProtocolError
+        io.close
+        retry
+      rescue Errno::EAGAIN, Errno::ECONNREFUSED
+        next
+      end
+    end
+    raise Remix::Stash::ClusterError,
+      "Unable to find suitable host to communicate with for #{key.inspect} (MD5-32=#{hash})"
+  end
+
+private
+
+  def connect(host, port)
+    socket = TCPSocket.new(host, port)
+    set_timeout(socket, 2)
+    socket
+  end
+
+  def host_to_io(key, host, port)
+    socket = @@connections[key] ||= connect(host, port)
+    return socket unless socket.closed?
+    @@connections.delete(key)
+    host_to_io(key, host, port)
+  end
+
+  def set_timeout(socket, seconds)
+    timeout = [seconds, 0].pack('l_2') # 2 seconds
+    socket.setsockopt(SOL_SOCKET, SO_SNDTIMEO, timeout)
+    socket.setsockopt(SOL_SOCKET, SO_RCVTIMEO, timeout)
+  end
+
+end

data/lib/remix/stash/extension.rb

@@ -0,0 +1,7 @@
+module Remix::Stash::Extension
+
+  def stash(name = :root)
+    Remix::Stash.new(name)
+  end
+
+end

data/lib/remix/stash/protocol.rb

@@ -0,0 +1,139 @@
+module Remix::Stash::Protocol
+  extend self
+
+  HEADER_FORMAT = "CCnCCnNNQ"
+
+  # Magic codes
+  REQUEST   = 0x80
+  RESPONSE = 0x81
+
+  # Command codes
+  GET         = 0x00
+  SET         = 0x01
+  ADD         = 0x02
+  REPLACE     = 0x03
+  DELETE      = 0x04
+  INCREMENT   = 0x05
+  DECREMENT   = 0x06
+  QUIT        = 0x07
+  FLUSH       = 0x08
+  GET_Q       = 0x09
+  NO_OP       = 0x0A
+  VERSION     = 0x0B
+  GET_K       = 0x0C
+  GET_K_Q     = 0x0D
+  APPEND      = 0x0E
+  PREPEND     = 0x0F
+  STAT        = 0x10
+  SET_Q       = 0x11
+  ADD_Q       = 0x12
+  REPLACE_Q   = 0x13
+  DELETE_Q    = 0x14
+  INCREMENT_Q = 0x15
+  DECREMENT_Q = 0x16
+  QUIT_Q      = 0x17
+  FLUSH_Q     = 0x18
+  APPEND_Q    = 0x19
+  PREPEND_Q   = 0x20
+
+  # Response codes
+  NO_ERROR                  = 0x0000
+  KEY_NOT_FOUND             = 0x0001
+  KEY_EXISTS                = 0x0002
+  VALUE_TOO_LARGE           = 0x0003
+  INVALID_ARGUMENTS         = 0x0004
+  ITEM_NOT_STORED           = 0x0005
+  INCR_ON_NON_NUMERIC_VALUE = 0x0006
+  UNKNOWN_COMMAND           = 0x0081
+  OUT_OF_MEMORY             = 0x0082
+
+  # Extras
+  COUNTER_FAULT_EXPIRATION = 0xFFFFFFFF
+
+  ADD_PACKET = HEADER_FORMAT + 'NNa*a*'
+  def add(io, key, data, ttl = 0)
+    header = [REQUEST, ADD, key.size, 8, 0, 0, data.size + key.size + 8, 0, 0, 0, ttl, key, data].pack(ADD_PACKET)
+    io.write(header)
+    resp = read_resp(io)
+    resp[:status] == NO_ERROR
+  end
+
+  DECR_PACKET = HEADER_FORMAT + 'NNQNa*'
+  def decr(io, key, step)
+    low, high = split64(step)
+    header = [REQUEST, DECREMENT, key.size, 20, 0, 0, key.size + 20, 0, 0, high, low, 0, COUNTER_FAULT_EXPIRATION, key].pack(DECR_PACKET)
+    io.write(header)
+    resp = read_resp(io)
+    if resp[:status] == NO_ERROR
+      parse_counter(resp[:body])
+    end
+  end
+
+  DELETE_PACKET = HEADER_FORMAT + 'a*'
+  def delete(io, key, ttl = 0)
+    header = [REQUEST, DELETE, key.size, 0, 0, 0, key.size, 0, 0, key].pack(DELETE_PACKET)
+    io.write(header)
+    resp = read_resp(io)
+    resp[:status] == NO_ERROR
+  end
+
+  FLUSH_PACKET = HEADER_FORMAT + 'N'
+  def flush(io)
+    header = [REQUEST, FLUSH, 0, 4, 0, 0, 4, 0, 0, 0].pack(FLUSH_PACKET)
+    io.write(header)
+    resp = read_resp(io)
+    resp[:status] == NO_ERROR
+  end
+
+  GET_PACKET = HEADER_FORMAT + 'a*'
+  GET_BODY = 4..-1
+  def get(io, key)
+    header = [REQUEST, GET, key.size, 0, 0, 0, key.size, 0, 0, key].pack(GET_PACKET)
+    io.write(header)
+    resp = read_resp(io)
+    resp[:status] == NO_ERROR ? resp[:body][GET_BODY] : nil
+  end
+
+  INCR_PACKET = HEADER_FORMAT + 'NNQNa*'
+  def incr(io, key, step)
+    low, high = split64(step)
+    header = [REQUEST, INCREMENT, key.size, 20, 0, 0, key.size + 20, 0, 0, high, low, 0, COUNTER_FAULT_EXPIRATION, key].pack(INCR_PACKET)
+    io.write(header)
+    resp = read_resp(io)
+    if resp[:status] == NO_ERROR
+      parse_counter(resp[:body])
+    end
+  end
+
+  SET_PACKET = HEADER_FORMAT + 'NNa*a*'
+  def set(io, key, data, ttl = 0)
+    header = [REQUEST, SET, key.size, 8, 0, 0, data.size + key.size + 8, 0, 0, 0, ttl, key, data].pack(SET_PACKET)
+    io << header
+    resp = read_resp(io)
+    resp[:status] == NO_ERROR
+  end
+
+private
+
+  COUNTER_SPLIT = 'NN'
+  def parse_counter(body)
+    a, b = body.unpack(COUNTER_SPLIT)
+    b | (a << 32)
+  end
+
+  RESP_HEADER = '@6nN'
+  def read_resp(io)
+    header = io.read(24)
+    header or raise Remix::Stash::ProtocolError,
+      "No data in response header"
+    status, body_length = *header.unpack(RESP_HEADER)
+    body_length.zero? ?
+      {:status => status} :
+      {:status => status, :body => io.read(body_length)}
+  end
+
+  def split64(n)
+    [0xFFFFFFFF & n, n >> 32]
+  end
+
+end