kestrel-client 0.6.1 → 0.6.4

data/VERSION

@@ -1 +1 @@
-0.6.1
+0.7.0

data/kestrel-client.gemspec

@@ -1,75 +1,71 @@
 # Generated by jeweler
 # DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = %q{kestrel-client}
-  s.version = "0.6.1"
+  s.name = "kestrel-client"
+  s.version = "0.6.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matt Freels", "Rael Dornfest"]
-  s.date = %q{2011-01-19}
-  s.description = %q{Ruby client for the Kestrel queue server}
-  s.email = %q{rael@twitter.com}
+  s.date = "2011-09-15"
+  s.description = "Ruby client for the Kestrel queue server"
+  s.email = "rael@twitter.com"
   s.extra_rdoc_files = [
     "README.md"
   ]
   s.files = [
-    ".gitignore",
-     "README.md",
-     "Rakefile",
-     "VERSION",
-     "kestrel-client.gemspec",
-     "lib/kestrel-client.rb",
-     "lib/kestrel.rb",
-     "lib/kestrel/client.rb",
-     "lib/kestrel/client/blocking.rb",
-     "lib/kestrel/client/envelope.rb",
-     "lib/kestrel/client/json.rb",
-     "lib/kestrel/client/namespace.rb",
-     "lib/kestrel/client/partitioning.rb",
-     "lib/kestrel/client/proxy.rb",
-     "lib/kestrel/client/retry_helper.rb",
-     "lib/kestrel/client/stats_helper.rb",
-     "lib/kestrel/client/transactional.rb",
-     "lib/kestrel/client/unmarshal.rb",
-     "lib/kestrel/config.rb",
-     "spec/kestrel/client/blocking_spec.rb",
-     "spec/kestrel/client/envelope_spec.rb",
-     "spec/kestrel/client/json_spec.rb",
-     "spec/kestrel/client/namespace_spec.rb",
-     "spec/kestrel/client/partitioning_spec.rb",
-     "spec/kestrel/client/transactional_spec.rb",
-     "spec/kestrel/client/unmarshal_spec.rb",
-     "spec/kestrel/client_spec.rb",
-     "spec/kestrel/config/kestrel.yml",
-     "spec/kestrel/config_spec.rb",
-     "spec/kestrel_benchmark.rb",
-     "spec/spec.opts",
-     "spec/spec_helper.rb"
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "kestrel-client.gemspec",
+    "lib/kestrel-client.rb",
+    "lib/kestrel.rb",
+    "lib/kestrel/client.rb",
+    "lib/kestrel/client/blocking.rb",
+    "lib/kestrel/client/envelope.rb",
+    "lib/kestrel/client/json.rb",
+    "lib/kestrel/client/namespace.rb",
+    "lib/kestrel/client/partitioning.rb",
+    "lib/kestrel/client/proxy.rb",
+    "lib/kestrel/client/stats_helper.rb",
+    "lib/kestrel/client/transactional.rb",
+    "lib/kestrel/client/unmarshal.rb",
+    "lib/kestrel/config.rb",
+    "spec/kestrel/client/blocking_spec.rb",
+    "spec/kestrel/client/envelope_spec.rb",
+    "spec/kestrel/client/json_spec.rb",
+    "spec/kestrel/client/namespace_spec.rb",
+    "spec/kestrel/client/partitioning_spec.rb",
+    "spec/kestrel/client/transactional_spec.rb",
+    "spec/kestrel/client/unmarshal_spec.rb",
+    "spec/kestrel/client_spec.rb",
+    "spec/kestrel/config/kestrel.yml",
+    "spec/kestrel/config_spec.rb",
+    "spec/kestrel_benchmark.rb",
+    "spec/spec.opts",
+    "spec/spec_helper.rb"
   ]
-  s.homepage = %q{http://github.com/freels/kestrel-client}
-  s.rdoc_options = ["--charset=UTF-8"]
+  s.homepage = "http://github.com/freels/kestrel-client"
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Ruby Kestrel client}
+  s.rubygems_version = "1.8.10"
+  s.summary = "Ruby Kestrel client"
   s.test_files = [
     "spec/kestrel/client/blocking_spec.rb",
-     "spec/kestrel/client/envelope_spec.rb",
-     "spec/kestrel/client/json_spec.rb",
-     "spec/kestrel/client/namespace_spec.rb",
-     "spec/kestrel/client/partitioning_spec.rb",
-     "spec/kestrel/client/transactional_spec.rb",
-     "spec/kestrel/client/unmarshal_spec.rb",
-     "spec/kestrel/client_spec.rb",
-     "spec/kestrel/config_spec.rb",
-     "spec/kestrel_benchmark.rb",
-     "spec/spec_helper.rb"
+    "spec/kestrel/client/envelope_spec.rb",
+    "spec/kestrel/client/json_spec.rb",
+    "spec/kestrel/client/namespace_spec.rb",
+    "spec/kestrel/client/partitioning_spec.rb",
+    "spec/kestrel/client/transactional_spec.rb",
+    "spec/kestrel/client/unmarshal_spec.rb",
+    "spec/kestrel/client_spec.rb",
+    "spec/kestrel/config_spec.rb",
+    "spec/kestrel_benchmark.rb",
+    "spec/spec_helper.rb"
   ]
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then

data/lib/kestrel/client.rb

@@ -1,7 +1,8 @@
+require 'forwardable'
+
 module Kestrel
-  class Client < Memcached
+  class Client
     require 'kestrel/client/stats_helper'
-    require 'kestrel/client/retry_helper'
 
     autoload :Proxy, 'kestrel/client/proxy'
     autoload :Envelope, 'kestrel/client/envelope'
@@ -35,12 +36,17 @@ module Kestrel
       Memcached::ServerError,
       Memcached::SystemError,
       Memcached::UnknownReadFailure,
-      Memcached::WriteFailure
+      Memcached::WriteFailure,
+      Memcached::NotFound
     ]
 
+    extend Forwardable
     include StatsHelper
-    include RetryHelper
 
+    attr_accessor :servers, :options
+    attr_reader :current_queue, :kestrel_options, :current_server
+
+    def_delegators :@write_client, :add, :append, :cas, :decr, :incr, :get_orig, :prepend
 
     def initialize(*servers)
       opts = servers.last.is_a?(Hash) ? servers.pop : {}
@@ -51,64 +57,44 @@ module Kestrel
       @gets_per_server = kestrel_options[:gets_per_server]
       @exception_retry_limit = kestrel_options[:exception_retry_limit]
       @counter = 0
+      @shuffle = true
 
       # we handle our own retries so that we can apply different
       # policies to sets and gets, so set memcached limit to 0
       opts[:exception_retry_limit] = 0
       opts[:distribution] = :random # force random distribution
 
-      super Array(servers).flatten.compact, opts
-    end
-
-
-    attr_reader :current_queue, :kestrel_options
-
+      self.servers  = Array(servers).flatten.compact
+      self.options  = opts
 
-    # Memcached overrides
-
-    %w(add append cas decr incr get_orig prepend).each do |m|
-      undef_method m
-    end
-
-    alias _super_get_from_random get
-    private :_super_get_from_random
-
-    def get_from_random(key, raw=false)
-      _super_get_from_random key, !raw
-    rescue Memcached::NotFound
+      @server_count = self.servers.size # Minor optimization.
+      @read_client  = Memcached.new(self.servers[rand(@server_count)], opts)
+      @write_client = Memcached.new(self.servers[rand(@server_count)], opts)
     end
 
-    # use get_from_last if available, otherwise redefine to point to
-    # plain old get
-    if method_defined? :get_from_last
-
-      def get_from_last(key, raw=false)
-        super key, !raw
-      rescue Memcached::NotFound
-      end
-
-    else
-
-      $stderr.puts "You have an older version of memcached.gem. Please upgrade to 0.19.6 or later for sticky get behavior."
-      def get_from_last(key, raw=false)
-        _super_get_from_random key, !raw
-      rescue Memcached::NotFound
-      end
-
-    end # end ifdef :)
-
     def delete(key, expiry=0)
-      with_retries { super key }
+      with_retries { @write_client.delete key }
     rescue Memcached::NotFound, Memcached::ServerEnd
     end
 
     def set(key, value, ttl=0, raw=false)
-      with_retries { super key, value, ttl, !raw }
+      with_retries { @write_client.set key, value, ttl, !raw }
       true
     rescue Memcached::NotStored
       false
     end
 
+    # This provides the necessary semantic to support transactionality
+    # in the Transactional client. It temporarily disables server
+    # shuffling to allow the client to close any open transactions on
+    # the current server before jumping.
+    #
+    def get_from_last(*args)
+      @shuffle = false
+      get *args
+    ensure
+      @shuffle = true
+    end
 
     # ==== Parameters
     # key<String>:: Queue name
@@ -125,12 +111,13 @@ module Kestrel
     #                 style" second argument.
     #
     def get(key, opts = {})
-      raw = opts.delete(:raw) || false
+      raw = opts[:raw] || false
       commands = extract_queue_commands(opts)
 
       val =
         begin
-          send(select_get_method(key), key + commands, raw)
+          shuffle_if_necessary! key
+          @read_client.get key + commands, !raw
         rescue *RECOVERABLE_ERRORS
           # we can't tell the difference between a server being down
           # and an empty queue, so just return nil. our sticky server
@@ -167,14 +154,18 @@ module Kestrel
       kestrel_opts
     end
 
-    def select_get_method(key)
-      if key != @current_queue || @counter >= @gets_per_server
+    def shuffle_if_necessary!(key)
+      # Don't reset servers on the first request:
+      # i.e. @counter == 0 && @current_queue == nil
+      if @shuffle &&
+          (@counter > 0 && key != @current_queue) ||
+          @counter >= @gets_per_server
         @counter = 0
         @current_queue = key
-        :get_from_random
+        @read_client.quit
+        @read_client.set_servers(servers[rand(@server_count)])
       else
         @counter +=1
-        :get_from_last
       end
     end
 
@@ -189,5 +180,20 @@ module Kestrel
 
       commands.map { |c| "/#{c}" }.join('')
     end
+
+    def with_retries #:nodoc:
+      yield
+    rescue *RECOVERABLE_ERRORS
+      tries ||= @exception_retry_limit + 1
+      tries -= 1
+      if tries > 0
+        retry
+      else
+        @write_client.quit
+        @write_client.set_servers(servers[rand(@server_count)])
+        raise
+      end
+    end
+
   end
 end

data/spec/kestrel/client_spec.rb

@@ -18,26 +18,30 @@ describe Kestrel::Client do
       end
 
       it "gets from the same server :gets_per_server times" do
-        mock(@kestrel).get_from_last("a_queue/t=10", false).times(100) { 'item' }
-        mock(@kestrel).get_from_random("a_queue/t=10", false).times(2) { 'item' }
+        client = @kestrel.instance_variable_get(:@read_client)
+        mock(client).get("a_queue/t=10", true).times(102).returns('item')
+        mock(client).quit.once
+        mock(client).set_servers.with(anything).once
 
         102.times { @kestrel.get("a_queue") }
       end
 
       it "gets from a different server when the last result was nil" do
-        mock(@kestrel).get_from_last("a_queue/t=10", false).never { nil }
-        mock(@kestrel).get_from_random("a_queue/t=10", false).times(3) { nil }
+        client = @kestrel.instance_variable_get(:@read_client)
+        mock(client).get("a_queue/t=10", true).returns(nil).twice
+        mock(client).quit.once
+        mock(client).set_servers.with(anything).once
 
-        3.times { @kestrel.get("a_queue") }
+        2.times { @kestrel.get("a_queue") }
       end
 
       it "returns nil if there is a recoverable exception" do
-        mock(@kestrel).select_get_method(@queue) { raise Memcached::SystemError }
+        mock(@kestrel).shuffle_if_necessary!(@queue) { raise Memcached::SystemError }
         @kestrel.get(@queue).should == nil
       end
 
       it "raises the exception if the exception is not recoverable" do
-        mock(@kestrel).select_get_method(@queue) { raise ArgumentError }
+        mock(@kestrel).shuffle_if_necessary!(@queue) { raise ArgumentError }
         lambda { @kestrel.get(@queue) }.should raise_error(ArgumentError)
       end
     end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kestrel-client
 version: !ruby/object:Gem::Version 
-  hash: 5
-  prerelease: false
+  hash: 15
+  prerelease: 
   segments: 
   - 0
   - 6
-  - 1
-  version: 0.6.1
+  - 4
+  version: 0.6.4
 platform: ruby
 authors: 
 - Matt Freels
@@ -16,8 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-19 00:00:00 -08:00
-default_executable: 
+date: 2011-09-15 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: memcached
@@ -44,7 +43,6 @@ extensions: []
 extra_rdoc_files: 
 - README.md
 files: 
-- .gitignore
 - README.md
 - Rakefile
 - VERSION
@@ -58,7 +56,6 @@ files:
 - lib/kestrel/client/namespace.rb
 - lib/kestrel/client/partitioning.rb
 - lib/kestrel/client/proxy.rb
-- lib/kestrel/client/retry_helper.rb
 - lib/kestrel/client/stats_helper.rb
 - lib/kestrel/client/transactional.rb
 - lib/kestrel/client/unmarshal.rb
@@ -76,13 +73,12 @@ files:
 - spec/kestrel_benchmark.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-has_rdoc: true
 homepage: http://github.com/freels/kestrel-client
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -106,7 +102,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Ruby Kestrel client

data/.gitignore

@@ -1 +0,0 @@
-pkg/*

data/lib/kestrel/client/retry_helper.rb

@@ -1,12 +0,0 @@
-module Kestrel::Client::RetryHelper
-
-  private
-
-  def with_retries #:nodoc:
-    yield
-  rescue *Kestrel::Client::RECOVERABLE_ERRORS
-    tries ||= @exception_retry_limit + 1
-    tries -= 1
-    tries > 0 ? retry : raise
-  end
-end