kestrel-client 0.4.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -12,6 +12,12 @@ Spec::Rake::SpecTask.new(:spec) do |t|
12
12
  t.spec_files = FileList['spec/**/*_spec.rb']
13
13
  end
14
14
 
15
+ desc "Run benchmarks"
16
+ Spec::Rake::SpecTask.new(:benchmark) do |t|
17
+ t.spec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
18
+ t.spec_files = [File.expand_path('spec/kestrel_benchmark.rb')]
19
+ end
20
+
15
21
  # gemification with jeweler
16
22
  begin
17
23
  require 'jeweler'
@@ -22,7 +28,7 @@ begin
22
28
  gemspec.email = "rael@twitter.com"
23
29
  gemspec.homepage = "http://github.com/freels/kestrel-client"
24
30
  gemspec.authors = ["Matt Freels", "Rael Dornfest"]
25
- gemspec.add_dependency 'memcached'
31
+ gemspec.add_dependency 'memcached', '>= 0.17'
26
32
  end
27
33
  Jeweler::GemcutterTasks.new
28
34
  rescue LoadError
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.4.1
1
+ 0.5.0
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{kestrel-client}
8
- s.version = "0.4.1"
8
+ s.version = "0.5.0"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Matt Freels", "Rael Dornfest"]
12
- s.date = %q{2010-08-05}
12
+ s.date = %q{2010-08-26}
13
13
  s.description = %q{Ruby client for the Kestrel queue server}
14
14
  s.email = %q{rael@twitter.com}
15
15
  s.extra_rdoc_files = [
@@ -31,7 +31,8 @@ Gem::Specification.new do |s|
31
31
  "lib/kestrel/client/partitioning.rb",
32
32
  "lib/kestrel/client/proxy.rb",
33
33
  "lib/kestrel/client/reliable.rb",
34
- "lib/kestrel/client/retrying.rb",
34
+ "lib/kestrel/client/retry_helper.rb",
35
+ "lib/kestrel/client/stats_helper.rb",
35
36
  "lib/kestrel/client/unmarshal.rb",
36
37
  "lib/kestrel/config.rb",
37
38
  "spec/kestrel/client/blocking_spec.rb",
@@ -40,11 +41,11 @@ Gem::Specification.new do |s|
40
41
  "spec/kestrel/client/namespace_spec.rb",
41
42
  "spec/kestrel/client/partitioning_spec.rb",
42
43
  "spec/kestrel/client/reliable_spec.rb",
43
- "spec/kestrel/client/retrying_spec.rb",
44
44
  "spec/kestrel/client/unmarshal_spec.rb",
45
45
  "spec/kestrel/client_spec.rb",
46
46
  "spec/kestrel/config/kestrel.yml",
47
47
  "spec/kestrel/config_spec.rb",
48
+ "spec/kestrel_benchmark.rb",
48
49
  "spec/spec.opts",
49
50
  "spec/spec_helper.rb"
50
51
  ]
@@ -60,10 +61,10 @@ Gem::Specification.new do |s|
60
61
  "spec/kestrel/client/namespace_spec.rb",
61
62
  "spec/kestrel/client/partitioning_spec.rb",
62
63
  "spec/kestrel/client/reliable_spec.rb",
63
- "spec/kestrel/client/retrying_spec.rb",
64
64
  "spec/kestrel/client/unmarshal_spec.rb",
65
65
  "spec/kestrel/client_spec.rb",
66
66
  "spec/kestrel/config_spec.rb",
67
+ "spec/kestrel_benchmark.rb",
67
68
  "spec/spec_helper.rb"
68
69
  ]
69
70
 
@@ -72,12 +73,12 @@ Gem::Specification.new do |s|
72
73
  s.specification_version = 3
73
74
 
74
75
  if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
75
- s.add_runtime_dependency(%q<memcached>, [">= 0"])
76
+ s.add_runtime_dependency(%q<memcached>, [">= 0.17"])
76
77
  else
77
- s.add_dependency(%q<memcached>, [">= 0"])
78
+ s.add_dependency(%q<memcached>, [">= 0.17"])
78
79
  end
79
80
  else
80
- s.add_dependency(%q<memcached>, [">= 0"])
81
+ s.add_dependency(%q<memcached>, [">= 0.17"])
81
82
  end
82
83
  end
83
84
 
@@ -1,5 +1,8 @@
1
1
  module Kestrel
2
- class Client < Memcached::Rails
2
+ class Client < Memcached
3
+ require 'kestrel/client/stats_helper'
4
+ require 'kestrel/client/retry_helper'
5
+
3
6
  autoload :Proxy, 'kestrel/client/proxy'
4
7
  autoload :Envelope, 'kestrel/client/envelope'
5
8
  autoload :Blocking, 'kestrel/client/blocking'
@@ -8,32 +11,89 @@ module Kestrel
8
11
  autoload :Namespace, 'kestrel/client/namespace'
9
12
  autoload :Json, 'kestrel/client/json'
10
13
  autoload :Reliable, "kestrel/client/reliable"
11
- autoload :Retrying, "kestrel/client/retrying"
12
14
 
13
- QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded}
15
+ KESTREL_OPTIONS = [:gets_per_server, :exception_retry_limit, :get_timeout_ms].freeze
14
16
 
15
- # ==== Parameters
16
- # key<String>:: Queue name
17
- # opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
18
- # allows collision-avoiding options support.
19
- #
20
- # ==== Options (opts)
21
- # :open<Boolean>:: Begins a reliable read.
22
- # :close<Boolean>:: Ends a reliable read.
23
- # :abort<Boolean>:: Cancels an existing reliable read
24
- # :peek<Boolean>:: Return the head of the queue, without removal
25
- # :timeout<Integer>:: Milliseconds to block for a new item
26
- # :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
27
- # style" second argument.
28
- #
29
- def get(key, opts = false)
30
- opts = extract_options(opts)
31
- raw = opts.delete(:raw)
32
- commands = extract_queue_commands(opts)
17
+ DEFAULT_OPTIONS = {
18
+ :retry_timeout => 0,
19
+ :exception_retry_limit => 5,
20
+ :timeout => 0.25,
21
+ :gets_per_server => 100,
22
+ :get_timeout_ms => 10
23
+ }.freeze
24
+
25
+ include StatsHelper
26
+ include RetryHelper
27
+
28
+
29
+ def initialize(*servers)
30
+ opts = servers.last.is_a?(Hash) ? servers.pop : {}
31
+ opts = DEFAULT_OPTIONS.merge(opts)
32
+
33
+ @kestrel_options = extract_kestrel_options!(opts)
34
+ @default_get_timeout = kestrel_options[:get_timeout_ms]
35
+ @gets_per_server = kestrel_options[:gets_per_server]
36
+ @exception_retry_limit = kestrel_options[:exception_retry_limit]
37
+ @counter = 0
38
+
39
+ # we handle our own retries so that we can apply different
40
+ # policies to sets and gets, so set memcached limit to 0
41
+ opts[:exception_retry_limit] = 0
42
+ opts[:distribution] = :random # force random distribution
43
+
44
+ super Array(servers).flatten.compact, opts
45
+ end
46
+
47
+
48
+ attr_reader :current_queue, :kestrel_options
49
+
50
+
51
+ # Memcached overrides
52
+
53
+ %w(add append cas decr incr get_orig prepend).each do |m|
54
+ undef_method m
55
+ end
56
+
57
+ alias _super_get_from_random get
58
+ private :_super_get_from_random
59
+
60
+ def get_from_random(key, raw=false)
61
+ _super_get_from_random key, !raw
62
+ rescue Memcached::NotFound
63
+ end
64
+
65
+ # use get_from_last if available, otherwise redefine to point to
66
+ # plain old get
67
+ if method_defined? :get_from_last
68
+
69
+ def get_from_last(key, raw=false)
70
+ super key, !raw
71
+ rescue Memcached::NotFound
72
+ end
33
73
 
34
- super key + commands, raw
74
+ else
75
+
76
+ $stderr.puts "You have an older version of memcached.gem. Please upgrade to 0.19.6 or later for sticky get behavior."
77
+ def get_from_last(key, raw=false)
78
+ _super_get_from_random key, !raw
79
+ rescue Memcached::NotFound
80
+ end
81
+
82
+ end # end ifdef :)
83
+
84
+ def delete(key, expiry=0)
85
+ with_retries { super key }
86
+ rescue Memcached::NotFound, Memcached::ServerEnd
35
87
  end
36
88
 
89
+ def set(key, value, ttl=0, raw=false)
90
+ with_retries { super key, value, ttl, !raw }
91
+ true
92
+ rescue Memcached::NotStored
93
+ false
94
+ end
95
+
96
+
37
97
  # ==== Parameters
38
98
  # key<String>:: Queue name
39
99
  # opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
@@ -48,20 +108,30 @@ module Kestrel
48
108
  # :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
49
109
  # style" second argument.
50
110
  #
51
- def get_from_last(key, opts = {})
52
- opts = extract_options(opts)
53
- raw = opts.delete(:raw)
111
+ def get(key, opts = {})
112
+ raw = opts.delete(:raw) || false
54
113
  commands = extract_queue_commands(opts)
55
114
 
56
- super key + commands, raw
115
+ val =
116
+ begin
117
+ send(select_get_method(key), key + commands, raw)
118
+ rescue Memcached::ATimeoutOccurred, Memcached::ServerIsMarkedDead
119
+ # we can't tell the difference between a server being down
120
+ # and an empty queue, so just return nil. our sticky server
121
+ # logic should eliminate piling on down servers
122
+ nil
123
+ end
124
+
125
+ # nil result, force next get to jump from current server
126
+ @counter = @gets_per_server unless val
127
+
128
+ val
57
129
  end
58
130
 
59
131
  def flush(queue)
60
132
  count = 0
61
133
  while sizeof(queue) > 0
62
- while get queue, :raw => true
63
- count += 1
64
- end
134
+ count += 1 while get queue, :raw => true
65
135
  end
66
136
  count
67
137
  end
@@ -70,27 +140,26 @@ module Kestrel
70
140
  get queue, :peek => true
71
141
  end
72
142
 
73
- def sizeof(queue)
74
- stat_info = stat(queue)
75
- stat_info ? stat_info['items'] : 0
76
- end
77
-
78
- def available_queues
79
- stats['queues'].keys.sort
80
- end
81
-
82
- def stats
83
- merge_stats(servers.map { |server| stats_for_server(server) })
84
- end
143
+ private
85
144
 
86
- def stat(queue)
87
- stats['queues'][queue]
145
+ def extract_kestrel_options!(opts)
146
+ kestrel_opts, memcache_opts = opts.inject([{}, {}]) do |(kestrel, memcache), (key, opt)|
147
+ (KESTREL_OPTIONS.include?(key) ? kestrel : memcache)[key] = opt
148
+ [kestrel, memcache]
149
+ end
150
+ opts.replace(memcache_opts)
151
+ kestrel_opts
88
152
  end
89
153
 
90
- private
91
-
92
- def extract_options(opts)
93
- opts.is_a?(Hash) ? opts : { :raw => !!opts }
154
+ def select_get_method(key)
155
+ if key != @current_queue || @counter >= @gets_per_server
156
+ @counter = 0
157
+ @current_queue = key
158
+ :get_from_random
159
+ else
160
+ @counter +=1
161
+ :get_from_last
162
+ end
94
163
  end
95
164
 
96
165
  def extract_queue_commands(opts)
@@ -98,66 +167,11 @@ module Kestrel
98
167
  opts[key]
99
168
  end
100
169
 
101
- commands << "t=#{opts[:timeout]}" if opts[:timeout]
102
-
103
- commands.map { |c| "/#{c}" }.join('')
104
- end
105
-
106
- def stats_for_server(server)
107
- server_name, port = server.split(/:/)
108
- socket = TCPSocket.new(server_name, port)
109
- socket.puts "STATS"
110
-
111
- stats = Hash.new
112
- stats['queues'] = Hash.new
113
- while line = socket.readline
114
- if line =~ /^STAT queue_(\S+?)_(#{QUEUE_STAT_NAMES.join("|")}) (\S+)/
115
- queue_name, queue_stat_name, queue_stat_value = $1, $2, deserialize_stat_value($3)
116
- stats['queues'][queue_name] ||= Hash.new
117
- stats['queues'][queue_name][queue_stat_name] = queue_stat_value
118
- elsif line =~ /^STAT (\w+) (\S+)/
119
- stat_name, stat_value = $1, deserialize_stat_value($2)
120
- stats[stat_name] = stat_value
121
- elsif line =~ /^END/
122
- socket.close
123
- break
124
- elsif defined?(RAILS_DEFAULT_LOGGER)
125
- RAILS_DEFAULT_LOGGER.debug("KestrelClient#stats_for_server: Ignoring #{line}")
126
- end
170
+ if timeout = (opts[:timeout] || @default_get_timeout)
171
+ commands << "t=#{timeout}"
127
172
  end
128
173
 
129
- stats
130
- end
131
-
132
- def merge_stats(all_stats)
133
- result = Hash.new
134
-
135
- all_stats.each do |stats|
136
- stats.each do |stat_name, stat_value|
137
- if result.has_key?(stat_name)
138
- if stat_value.kind_of?(Hash)
139
- result[stat_name] = merge_stats([result[stat_name], stat_value])
140
- else
141
- result[stat_name] += stat_value
142
- end
143
- else
144
- result[stat_name] = stat_value
145
- end
146
- end
147
- end
148
-
149
- result
150
- end
151
-
152
- def deserialize_stat_value(value)
153
- case value
154
- when /^\d+\.\d+$/:
155
- value.to_f
156
- when /^\d+$/:
157
- value.to_i
158
- else
159
- value
160
- end
174
+ commands.map { |c| "/#{c}" }.join('')
161
175
  end
162
176
  end
163
177
  end
@@ -1,15 +1,21 @@
1
1
  module Kestrel
2
2
  class Client
3
3
  class Blocking < Proxy
4
- DEFAULT_TIMEOUT = 1000
5
4
 
6
- def get(key, opts = false)
7
- opts = extract_options(opts)
8
- opts[:timeout] = DEFAULT_TIMEOUT
5
+ # random backoff sleeping
9
6
 
10
- loop do
11
- response = client.get(key, opts)
12
- return response if response
7
+ SLEEP_TIMES = [[0] * 1, [0.01] * 2, [0.1] * 2, [0.5] * 2, [1.0] * 1].flatten
8
+
9
+ def get(*args)
10
+ count = 0
11
+
12
+ while count += 1
13
+
14
+ if response = client.get(*args)
15
+ return response
16
+ end
17
+
18
+ sleep_for_count(count)
13
19
  end
14
20
  end
15
21
 
@@ -17,6 +23,14 @@ module Kestrel
17
23
  client.get(*args)
18
24
  end
19
25
 
26
+ private
27
+
28
+ def sleep_for_count(count)
29
+ base = SLEEP_TIMES[count] || SLEEP_TIMES.last
30
+
31
+ time = ((rand * base) + base) / 2
32
+ sleep time if time > 0
33
+ end
20
34
  end
21
35
  end
22
36
  end
@@ -1,136 +1,107 @@
1
- module Kestrel
2
- class Client
3
- #--
4
- # TODO: Pull out the sticky server logic into Client. This class
5
- # should only be responsible for the retry semantics.
6
- # TODO: Ensure that errors are pushed onto the error queue on the
7
- # same server on which the error occurred.
8
- #++
9
- class Reliable < Proxy
10
- # Raised when a caller attempts to use this proxy across
11
- # multiple queues.
12
- class MultipleQueueException < StandardError; end
13
-
14
- class RetryableJob < Struct.new(:retries, :job); end
15
-
16
- # Number of times to retry a job before giving up
17
- DEFAULT_RETRIES = 100
18
-
19
- # Pct. of the time during 'normal' processing we check the error queue first
20
- ERROR_PROCESSING_RATE = 0.1
21
-
22
- # Maximum number of gets to execute before switching servers
23
- MAX_PER_SERVER = 100_000
24
-
25
- # ==== Parameters
26
- # client<Kestrel::Client>:: Client
27
- # retry_count<Integer>:: Number of times to retry a job before
28
- # giving up. Defaults to DEFAULT_RETRIES
29
- # error_rate<Float>:: Pct. of the time during 'normal'
30
- # processing we check the error queue
31
- # first. Defaults to ERROR_PROCESSING_RATE
32
- # per_server<Integer>:: Number of gets to execute against a
33
- # single server, before changing
34
- # servers. Defaults to MAX_PER_SERVER
35
- #
36
- def initialize(client, retry_count = nil, error_rate = nil, per_server = nil)
37
- @retry_count = retry_count || DEFAULT_RETRIES
38
- @error_rate = error_rate || ERROR_PROCESSING_RATE
39
- @per_server = per_server || MAX_PER_SERVER
40
- @counter = 0 # Command counter
41
- super(client)
42
- end
43
-
44
- # Returns job from the +key+ queue 1 - ERROR_PROCESSING_RATE
45
- # pct. of the time. Every so often, checks the error queue for
46
- # jobs and returns a retryable job. If either the error queue or
47
- # +key+ queue are empty, attempts to pull a job from the
48
- # alternate queue before giving up.
49
- #
50
- # ==== Returns
51
- # Job, possibly retryable, or nil
52
- #
53
- def get(key, opts = false)
54
- raise MultipleQueueException if @key && key != @key
55
-
56
- job =
57
- if rand < @error_rate
58
- get_with_fallback(key + "_errors", key, opts)
59
- else
60
- get_with_fallback(key, key + "_errors", opts)
61
- end
62
-
63
- if job
64
- @key = key
65
- @job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
66
- @job.job
67
- else
68
- @key = @job = nil
69
- end
70
- end
71
-
72
- def current_try
73
- @job ? @job.retries + 1 : 1
74
- end
75
-
76
- # Enqueues the current job on the error queue for later
77
- # retry. If the job has been retried DEFAULT_RETRIES times,
78
- # gives up entirely.
79
- #
80
- # ==== Returns
81
- # Boolean:: true if the job is retryable, false otherwise
82
- #
83
- def retry
84
- return unless @job
85
-
86
- close_open_transaction!
87
- @job.retries += 1
88
-
89
- if @job.retries < @retry_count
90
- client.set(@key + "_errors", @job)
91
- true
92
- else
93
- false
94
- end
95
- end
96
-
97
- private
98
-
99
- # If a get against the +primary+ queue is nil, falls back to the
100
- # +secondary+ queue.
101
- #
102
- # Also, this executes a get on the first request, then a get_from_last
103
- # on each ensuing request for @per_server requests. This keeps the
104
- # client "attached" to a single server for a period of time.
105
- #
106
- def get_with_fallback(primary, secondary, opts) #:nodoc:
107
- opts = extract_options(opts)
108
- opts.merge! :close => true, :open => true
109
-
110
- if @counter == 0
111
- close_open_transaction! if @job
112
- @counter += 1
113
- command = :get
114
- elsif @counter < @per_server
115
- # Open transactions are implicitly closed, here.
116
- @counter += 1
117
- command = :get_from_last
118
- else
119
- close_open_transaction! if @job
120
- @counter = 0
121
- command = :get
122
- end
123
-
124
- client.send(command, primary, opts) || client.send(command, secondary, opts)
125
- end
126
-
127
- def close_open_transaction! #:nodoc:
128
- if @job.retries == 0
129
- client.get_from_last(@key, :close => true, :open => false)
130
- else
131
- client.get_from_last(@key + "_errors", :close => true, :open => false)
132
- end
133
- end
1
+ class Kestrel::Client::Reliable < Kestrel::Client::Proxy
2
+
3
+ # Raised when a caller attempts to use this proxy across
4
+ # multiple queues.
5
+ class MultipleQueueException < StandardError; end
6
+
7
+
8
+ class RetryableJob < Struct.new(:retries, :job); end
9
+
10
+
11
+ # Number of times to retry a job before giving up
12
+ DEFAULT_RETRIES = 100
13
+
14
+
15
+ # Pct. of the time during 'normal' processing we check the error queue first
16
+ ERROR_PROCESSING_RATE = 0.1
17
+
18
+
19
+ # Maximum number of gets to execute before switching servers
20
+ MAX_PER_SERVER = 100_000
21
+
22
+
23
+ # ==== Parameters
24
+ # client<Kestrel::Client>:: Client
25
+ # max_retries<Integer>:: Number of times to retry a job before
26
+ # giving up. Defaults to DEFAULT_RETRIES
27
+ # error_rate<Float>:: Pct. of the time during 'normal'
28
+ # processing we check the error queue
29
+ # first. Defaults to ERROR_PROCESSING_RATE
30
+ # per_server<Integer>:: Number of gets to execute against a
31
+ # single server, before changing
32
+ # servers. Defaults to MAX_PER_SERVER
33
+ #
34
+ def initialize(client, max_retries = nil, error_rate = nil, per_server = nil)
35
+ @max_retries = max_retries || DEFAULT_RETRIES
36
+ @error_rate = error_rate || ERROR_PROCESSING_RATE
37
+ @per_server = per_server || MAX_PER_SERVER
38
+ @counter = 0 # Command counter
39
+ super(client)
40
+ end
41
+
42
+ attr_reader :current_queue
43
+
44
+ # Returns job from the +key+ queue 1 - ERROR_PROCESSING_RATE
45
+ # pct. of the time. Every so often, checks the error queue for
46
+ # jobs and returns a retryable job. If either the error queue or
47
+ # +key+ queue are empty, attempts to pull a job from the
48
+ # alternate queue before giving up.
49
+ #
50
+ # ==== Returns
51
+ # Job, possibly retryable, or nil
52
+ #
53
+ def get(key, opts = {})
54
+ raise MultipleQueueException if current_queue && key != current_queue
55
+
56
+ close_transaction(current_try == 1 ? key : "#{key}_errors")
57
+
58
+ q1, q2 = (rand < @error_rate) ? [key + "_errors", key] : [key, key + "_errors"]
59
+
60
+ if job = get_with_fallback(q1, q2, opts.merge(:close => true, :open => true))
61
+ @current_queue = key
62
+ @job = job.is_a?(RetryableJob) ? job : RetryableJob.new(0, job)
63
+ @job.job
64
+ else
65
+ @current_queue = @job = nil
134
66
  end
135
67
  end
68
+
69
+ def current_try
70
+ @job ? @job.retries + 1 : 1
71
+ end
72
+
73
+ # Enqueues the current job on the error queue for later
74
+ # retry. If the job has been retried DEFAULT_RETRIES times,
75
+ # gives up entirely.
76
+ #
77
+ # ==== Returns
78
+ # Boolean:: true if the job is retryable, false otherwise
79
+ #
80
+ def retry
81
+ return unless @job
82
+
83
+ @job.retries += 1
84
+
85
+ if should_retry = @job.retries < @max_retries
86
+ client.set(current_queue + "_errors", @job)
87
+ end
88
+
89
+ # close the transaction on the original queue if this is the first retry
90
+ close_transaction(@job.retries == 1 ? current_queue : "#{current_queue}_errors")
91
+
92
+ should_retry
93
+ end
94
+
95
+ private
96
+
97
+ # If a get against the +primary+ queue is nil, falls back to the
98
+ # +secondary+ queue.
99
+ #
100
+ def get_with_fallback(primary, secondary, opts) #:nodoc:
101
+ client.get(primary, opts) || client.get(secondary, opts)
102
+ end
103
+
104
+ def close_transaction(key) #:nodoc:
105
+ client.get_from_last("#{key}/close")
106
+ end
136
107
  end
@@ -0,0 +1,28 @@
1
+ module Kestrel::Client::RetryHelper
2
+
3
+ # Exceptions which are connection failures we retry after
4
+ RECOVERABLE_ERRORS = [
5
+ Memcached::ServerIsMarkedDead,
6
+ Memcached::ATimeoutOccurred,
7
+ Memcached::ConnectionBindFailure,
8
+ Memcached::ConnectionFailure,
9
+ Memcached::ConnectionSocketCreateFailure,
10
+ Memcached::Failure,
11
+ Memcached::MemoryAllocationFailure,
12
+ Memcached::ReadFailure,
13
+ Memcached::ServerError,
14
+ Memcached::SystemError,
15
+ Memcached::UnknownReadFailure,
16
+ Memcached::WriteFailure
17
+ ]
18
+
19
+ private
20
+
21
+ def with_retries #:nodoc:
22
+ yield
23
+ rescue *RECOVERABLE_ERRORS
24
+ tries ||= @exception_retry_limit + 1
25
+ tries -= 1
26
+ tries > 0 ? retry : raise
27
+ end
28
+ end
@@ -0,0 +1,80 @@
1
+ module Kestrel::Client::StatsHelper
2
+
3
+ QUEUE_STAT_NAMES = %w{items bytes total_items logsize expired_items mem_items mem_bytes age discarded}
4
+
5
+ def sizeof(queue)
6
+ stat_info = stat(queue)
7
+ stat_info ? stat_info['items'] : 0
8
+ end
9
+
10
+ def available_queues
11
+ stats['queues'].keys.sort
12
+ end
13
+
14
+ def stats
15
+ merge_stats(servers.map { |server| stats_for_server(server) })
16
+ end
17
+
18
+ def stat(queue)
19
+ stats['queues'][queue]
20
+ end
21
+
22
+ private
23
+
24
+ def stats_for_server(server)
25
+ server_name, port = server.split(/:/)
26
+ socket = TCPSocket.new(server_name, port)
27
+ socket.puts "STATS"
28
+
29
+ stats = Hash.new
30
+ stats['queues'] = Hash.new
31
+ while line = socket.readline
32
+ if line =~ /^STAT queue_(\S+?)_(#{QUEUE_STAT_NAMES.join("|")}) (\S+)/
33
+ queue_name, queue_stat_name, queue_stat_value = $1, $2, deserialize_stat_value($3)
34
+ stats['queues'][queue_name] ||= Hash.new
35
+ stats['queues'][queue_name][queue_stat_name] = queue_stat_value
36
+ elsif line =~ /^STAT (\w+) (\S+)/
37
+ stat_name, stat_value = $1, deserialize_stat_value($2)
38
+ stats[stat_name] = stat_value
39
+ elsif line =~ /^END/
40
+ socket.close
41
+ break
42
+ elsif defined?(RAILS_DEFAULT_LOGGER)
43
+ RAILS_DEFAULT_LOGGER.debug("KestrelClient#stats_for_server: Ignoring #{line}")
44
+ end
45
+ end
46
+
47
+ stats
48
+ end
49
+
50
+ def merge_stats(all_stats)
51
+ result = Hash.new
52
+
53
+ all_stats.each do |stats|
54
+ stats.each do |stat_name, stat_value|
55
+ if result.has_key?(stat_name)
56
+ if stat_value.kind_of?(Hash)
57
+ result[stat_name] = merge_stats([result[stat_name], stat_value])
58
+ else
59
+ result[stat_name] += stat_value
60
+ end
61
+ else
62
+ result[stat_name] = stat_value
63
+ end
64
+ end
65
+ end
66
+
67
+ result
68
+ end
69
+
70
+ def deserialize_stat_value(value)
71
+ case value
72
+ when /^\d+\.\d+$/:
73
+ value.to_f
74
+ when /^\d+$/:
75
+ value.to_i
76
+ else
77
+ value
78
+ end
79
+ end
80
+ end
@@ -1,9 +1,10 @@
1
1
  module Kestrel
2
2
  class Client
3
3
  class Unmarshal < Proxy
4
- def get(keys, raw = false)
5
- response = client.get(keys, true)
6
- return response if raw
4
+ def get(key, opts = {})
5
+ response = client.get(key, opts.merge(:raw => true))
6
+ return response if opts[:raw]
7
+
7
8
  if is_marshaled?(response)
8
9
  Marshal.load_with_constantize(response, loaded_constants = [])
9
10
  else
@@ -14,8 +14,7 @@ describe "Kestrel::Client::Blocking" do
14
14
 
15
15
  it "blocks on a get until the get works" do
16
16
  mock(@raw_kestrel_client).
17
- get(@queue, :raw => false, :timeout => Kestrel::Client::Blocking::DEFAULT_TIMEOUT) { nil }.then.
18
- get(@queue, :raw => false, :timeout => Kestrel::Client::Blocking::DEFAULT_TIMEOUT) { :mcguffin }
17
+ get(@queue) { nil }.times(5).then.get(@queue) { :mcguffin }
19
18
  @kestrel.get(@queue).should == :mcguffin
20
19
  end
21
20
 
@@ -1,25 +1,37 @@
1
1
  require 'spec/spec_helper'
2
2
 
3
- class Envelope; end
3
+ class Envelope
4
+ class << self; attr_accessor :unwraps end
5
+
6
+ def initialize(item); @item = item end
7
+ def unwrap; self.class.unwraps += 1; @item end
8
+ end
4
9
 
5
10
  describe Kestrel::Client::Envelope do
6
11
  describe "Instance Methods" do
7
12
  before do
13
+ Envelope.unwraps = 0
8
14
  @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
9
15
  @kestrel = Kestrel::Client::Envelope.new(Envelope, @raw_kestrel_client)
10
16
  end
11
17
 
12
18
  describe "#get and #set" do
13
19
  describe "envelopes" do
20
+ it "integrates" do
21
+ @kestrel.set("a_queue", :mcguffin)
22
+ @kestrel.get("a_queue").should == :mcguffin
23
+ Envelope.unwraps.should == 1
24
+ end
25
+
14
26
  it "creates an envelope on a set" do
15
27
  mock(Envelope).new(:mcguffin)
16
28
  @kestrel.set('a_queue', :mcguffin)
17
29
  end
18
30
 
19
31
  it "unwraps an envelope on a get" do
20
- envelope = Envelope.new
32
+ envelope = Envelope.new(:mcguffin)
21
33
  mock(@raw_kestrel_client).get('a_queue') { envelope }
22
- mock(envelope).unwrap { :mcguffin }
34
+ mock.proxy(envelope).unwrap
23
35
  @kestrel.get('a_queue').should == :mcguffin
24
36
  end
25
37
 
@@ -1,33 +1,6 @@
1
1
  require 'spec/spec_helper'
2
2
 
3
3
  describe "Kestrel::Client::Reliable" do
4
- describe "Sticky" do
5
- before do
6
- @max_requests = 2
7
- @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
8
- @kestrel = Kestrel::Client::Reliable.new(@raw_kestrel_client, nil, nil, @max_requests)
9
- stub(@kestrel).rand { 1 }
10
- @queue = "some_queue"
11
- end
12
-
13
- describe "#get" do
14
-
15
- it 'does a get on the first request' do
16
- mock(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
17
- @kestrel.get(@queue)
18
- end
19
-
20
- it 'does a get_from_last a number of times, then a get' do
21
- mock(@raw_kestrel_client).get(@queue, anything).twice { :mcguffin }
22
- mock(@raw_kestrel_client).get_from_last(@queue, anything).twice { :mcguffin }
23
-
24
- @kestrel.get(@queue) # Initial get
25
- @kestrel.get(@queue) # get_from_last
26
- @kestrel.get(@queue) # get_from_last txn close, get
27
- end
28
- end
29
- end
30
-
31
4
  describe "Instance Methods" do
32
5
  before do
33
6
  @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
@@ -39,7 +12,7 @@ describe "Kestrel::Client::Reliable" do
39
12
  describe "#get" do
40
13
 
41
14
  it "asks for a transaction" do
42
- mock(@raw_kestrel_client).get(@queue, :raw => false, :open => true, :close => true) { :mcguffin }
15
+ mock(@raw_kestrel_client).get(@queue, :open => true, :close => true) { :mcguffin }
43
16
  @kestrel.get(@queue).should == :mcguffin
44
17
  end
45
18
 
@@ -92,7 +65,7 @@ describe "Kestrel::Client::Reliable" do
92
65
  stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
93
66
  @kestrel.get(@queue)
94
67
 
95
- mock(@raw_kestrel_client).get_from_last(@queue, :close => true, :open => false)
68
+ mock(@raw_kestrel_client).get_from_last(@queue + "/close")
96
69
  @kestrel.get(@queue)
97
70
  end
98
71
 
@@ -103,7 +76,7 @@ describe "Kestrel::Client::Reliable" do
103
76
  end
104
77
  @kestrel.get(@queue)
105
78
 
106
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true, :open => false)
79
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
107
80
  @kestrel.get(@queue)
108
81
  end
109
82
 
@@ -184,7 +157,7 @@ describe "Kestrel::Client::Reliable" do
184
157
  stub(@raw_kestrel_client).get(@queue, anything) { :mcguffin }
185
158
  @kestrel.get(@queue)
186
159
 
187
- mock(@raw_kestrel_client).get_from_last(@queue, :close => true, :open => false)
160
+ mock(@raw_kestrel_client).get_from_last(@queue + "/close")
188
161
  @kestrel.retry
189
162
  end
190
163
 
@@ -195,7 +168,7 @@ describe "Kestrel::Client::Reliable" do
195
168
  end
196
169
  @kestrel.get(@queue)
197
170
 
198
- mock(@raw_kestrel_client).get_from_last(@queue + "_errors", :close => true, :open => false)
171
+ mock(@raw_kestrel_client).get_from_last(@queue + "_errors/close")
199
172
  @kestrel.retry
200
173
  end
201
174
 
@@ -8,26 +8,31 @@ describe Kestrel::Client::Unmarshal do
8
8
  end
9
9
 
10
10
  describe "#get" do
11
+ it "integrates" do
12
+ @kestrel.set('a_queue', "foo")
13
+ @kestrel.get('a_queue').should == 'foo'
14
+ end
15
+
11
16
  it "unmarshals marshaled objects" do
12
17
  test_object = {:a => 1, :b => [1, 2, 3]}
13
- mock(@raw_kestrel_client).get('a_queue', true) { Marshal.dump(test_object) }
18
+ mock(@raw_kestrel_client).get('a_queue', :raw => true) { Marshal.dump(test_object) }
14
19
  @kestrel.get('a_queue').should == test_object
15
20
  end
16
21
 
17
22
  it "does not unmarshal when raw is true" do
18
23
  test_object = {:a => 1, :b => [1, 2, 3]}
19
- mock(@raw_kestrel_client).get('a_queue', true) { Marshal.dump(test_object) }
20
- @kestrel.get('a_queue', true).should == Marshal.dump(test_object)
24
+ mock(@raw_kestrel_client).get('a_queue', :raw => true) { Marshal.dump(test_object) }
25
+ @kestrel.get('a_queue', :raw => true).should == Marshal.dump(test_object)
21
26
  end
22
27
 
23
- it "pasess through objects" do
28
+ it "passes through objects" do
24
29
  test_object = Object.new
25
- mock(@raw_kestrel_client).get('a_queue', true) { test_object }
30
+ mock(@raw_kestrel_client).get('a_queue', :raw => true) { test_object }
26
31
  @kestrel.get('a_queue').should == test_object
27
32
  end
28
33
 
29
34
  it "passes through strings" do
30
- mock(@raw_kestrel_client).get('a_queue', true) { "I am not marshaled" }
35
+ mock(@raw_kestrel_client).get('a_queue', :raw => true) { "I am not marshaled" }
31
36
  @kestrel.get('a_queue').should == "I am not marshaled"
32
37
  end
33
38
  end
@@ -3,8 +3,7 @@ require 'spec/spec_helper'
3
3
  describe Kestrel::Client do
4
4
  describe "Instance Methods" do
5
5
  before do
6
- @kestrel = Kestrel::Client.new(*Kestrel::Config.default)
7
- stub(@kestrel).with_timing(anything) { |_, block| block.call }
6
+ @kestrel = Kestrel::Client.new('localhost:22133')
8
7
  end
9
8
 
10
9
  describe "#get and #set" do
@@ -13,6 +12,36 @@ describe Kestrel::Client do
13
12
  @kestrel.set(queue, value = "russell's reserve")
14
13
  @kestrel.get(queue).should == value
15
14
  end
15
+
16
+ it "returns nil when getting from a queue that does not exist" do
17
+ @kestrel.get('nonexistent_queue').should == nil
18
+ end
19
+
20
+ it "gets from the same server :gets_per_server times" do
21
+ mock(@kestrel).get_from_last("a_queue/t=10", false).times(100) { 'item' }
22
+ mock(@kestrel).get_from_random("a_queue/t=10", false).times(2) { 'item' }
23
+
24
+ 102.times { @kestrel.get("a_queue") }
25
+ end
26
+
27
+ it "gets from a different server when the last result was nil" do
28
+ mock(@kestrel).get_from_last("a_queue/t=10", false).never { nil }
29
+ mock(@kestrel).get_from_random("a_queue/t=10", false).times(3) { nil }
30
+
31
+ 3.times { @kestrel.get("a_queue") }
32
+ end
33
+ end
34
+
35
+ describe "retry behavior" do
36
+ it "does not retry gets" do
37
+ mock(@kestrel).with_retries.never
38
+ @kestrel.get("a_queue")
39
+ end
40
+
41
+ it "retries sets" do
42
+ mock(@kestrel).with_retries
43
+ @kestrel.set("a_queue", "value")
44
+ end
16
45
  end
17
46
 
18
47
  describe "#flush" do
@@ -42,6 +71,33 @@ describe Kestrel::Client do
42
71
  end
43
72
  end
44
73
 
74
+ describe "#with_retries" do
75
+ it "retries a specified number of times" do
76
+ mock(@kestrel).set(anything, anything) { raise Memcached::SystemError }.times(6)
77
+
78
+ lambda do
79
+ @kestrel.send(:with_retries) { @kestrel.set("a_queue", "foo") }
80
+ end.should raise_error(Memcached::SystemError)
81
+ end
82
+
83
+ it "does not raise if within the retry limit" do
84
+ mock(@kestrel).set(anything, anything) { raise Memcached::SystemError }.times(5).
85
+ then.set(anything, anything) { true }
86
+
87
+ lambda do
88
+ @kestrel.send(:with_retries) { @kestrel.set("a_queue", "foo") }
89
+ end.should_not raise_error(Memcached::SystemError)
90
+ end
91
+
92
+ it "does not catch unknown errors" do
93
+ mock(@kestrel).set(anything, anything) { raise ArgumentError }
94
+
95
+ lambda do
96
+ @kestrel.send(:with_retries) { @kestrel.set("a_queue", "foo") }
97
+ end.should raise_error(ArgumentError)
98
+ end
99
+ end
100
+
45
101
  describe "#stats" do
46
102
  it "retrieves stats" do
47
103
  @kestrel.set("test-queue-name", 97)
@@ -0,0 +1,26 @@
1
+ require 'spec/spec_helper'
2
+ require 'benchmark'
3
+
4
+ describe Kestrel::Client do
5
+ before do
6
+ @queue = "a_queue"
7
+ @kestrel = Kestrel::Client.new(*Kestrel::Config.default)
8
+
9
+ @kestrel.delete(@queue) rescue nil # Memcache::ServerEnd bug
10
+ end
11
+
12
+ it "is fast" do
13
+ @kestrel.flush(@queue)
14
+ @value = { :value => "a value" }
15
+ @raw_value = Marshal.dump(@value)
16
+
17
+ times = 10_000
18
+
19
+ Benchmark.bm do |x|
20
+ x.report("set:") { for i in 1..times; @kestrel.set(@queue, @value); end }
21
+ x.report("get:") { for i in 1..times; @kestrel.get(@queue); end }
22
+ x.report("set (raw):") { for i in 1..times; @kestrel.set(@queue, @raw_value, 0, true); end }
23
+ x.report("get (raw):") { for i in 1..times; @kestrel.get(@queue, true); end }
24
+ end
25
+ end
26
+ end
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kestrel-client
3
3
  version: !ruby/object:Gem::Version
4
- hash: 13
4
+ hash: 11
5
5
  prerelease: false
6
6
  segments:
7
7
  - 0
8
- - 4
9
- - 1
10
- version: 0.4.1
8
+ - 5
9
+ - 0
10
+ version: 0.5.0
11
11
  platform: ruby
12
12
  authors:
13
13
  - Matt Freels
@@ -16,7 +16,7 @@ autorequire:
16
16
  bindir: bin
17
17
  cert_chain: []
18
18
 
19
- date: 2010-08-05 00:00:00 -07:00
19
+ date: 2010-08-26 00:00:00 -07:00
20
20
  default_executable:
21
21
  dependencies:
22
22
  - !ruby/object:Gem::Dependency
@@ -27,10 +27,11 @@ dependencies:
27
27
  requirements:
28
28
  - - ">="
29
29
  - !ruby/object:Gem::Version
30
- hash: 3
30
+ hash: 41
31
31
  segments:
32
32
  - 0
33
- version: "0"
33
+ - 17
34
+ version: "0.17"
34
35
  type: :runtime
35
36
  version_requirements: *id001
36
37
  description: Ruby client for the Kestrel queue server
@@ -57,7 +58,8 @@ files:
57
58
  - lib/kestrel/client/partitioning.rb
58
59
  - lib/kestrel/client/proxy.rb
59
60
  - lib/kestrel/client/reliable.rb
60
- - lib/kestrel/client/retrying.rb
61
+ - lib/kestrel/client/retry_helper.rb
62
+ - lib/kestrel/client/stats_helper.rb
61
63
  - lib/kestrel/client/unmarshal.rb
62
64
  - lib/kestrel/config.rb
63
65
  - spec/kestrel/client/blocking_spec.rb
@@ -66,11 +68,11 @@ files:
66
68
  - spec/kestrel/client/namespace_spec.rb
67
69
  - spec/kestrel/client/partitioning_spec.rb
68
70
  - spec/kestrel/client/reliable_spec.rb
69
- - spec/kestrel/client/retrying_spec.rb
70
71
  - spec/kestrel/client/unmarshal_spec.rb
71
72
  - spec/kestrel/client_spec.rb
72
73
  - spec/kestrel/config/kestrel.yml
73
74
  - spec/kestrel/config_spec.rb
75
+ - spec/kestrel_benchmark.rb
74
76
  - spec/spec.opts
75
77
  - spec/spec_helper.rb
76
78
  has_rdoc: true
@@ -114,8 +116,8 @@ test_files:
114
116
  - spec/kestrel/client/namespace_spec.rb
115
117
  - spec/kestrel/client/partitioning_spec.rb
116
118
  - spec/kestrel/client/reliable_spec.rb
117
- - spec/kestrel/client/retrying_spec.rb
118
119
  - spec/kestrel/client/unmarshal_spec.rb
119
120
  - spec/kestrel/client_spec.rb
120
121
  - spec/kestrel/config_spec.rb
122
+ - spec/kestrel_benchmark.rb
121
123
  - spec/spec_helper.rb
@@ -1,51 +0,0 @@
1
- module Kestrel
2
- class Client
3
- class Retrying < Proxy
4
-
5
- # Number of times to retry after connection failures
6
- DEFAULT_RETRY_COUNT = 5
7
-
8
- # Exceptions which are connection failures we retry after
9
- RECOVERABLE_ERRORS = [
10
- Memcached::ServerIsMarkedDead,
11
- Memcached::ATimeoutOccurred,
12
- Memcached::ConnectionBindFailure,
13
- Memcached::ConnectionFailure,
14
- Memcached::ConnectionSocketCreateFailure,
15
- Memcached::Failure,
16
- Memcached::MemoryAllocationFailure,
17
- Memcached::ReadFailure,
18
- Memcached::ServerError,
19
- Memcached::SystemError,
20
- Memcached::UnknownReadFailure,
21
- Memcached::WriteFailure
22
- ]
23
-
24
- def initialize(client, retry_count = nil)
25
- @retry_count = retry_count || DEFAULT_RETRY_COUNT
26
- super(client)
27
- end
28
-
29
- %w(set get delete).each do |method|
30
- class_eval "def #{method}(*args); retry_call(#{method.inspect}, *args) end", __FILE__, __LINE__
31
- end
32
-
33
- private
34
-
35
- def retry_call(method, *args) #:nodoc:
36
- begin
37
- tries ||= 0
38
- client.send(method, *args)
39
- rescue *RECOVERABLE_ERRORS
40
- if tries < @retry_count
41
- tries += 1
42
- retry
43
- else
44
- raise
45
- end
46
- end
47
- end
48
-
49
- end
50
- end
51
- end
@@ -1,46 +0,0 @@
1
- require 'spec/spec_helper'
2
-
3
- describe Kestrel::Client::Retrying do
4
- before do
5
- @raw_kestrel_client = Kestrel::Client.new(*Kestrel::Config.default)
6
- @kestrel = Kestrel::Client::Retrying.new(@raw_kestrel_client)
7
- @queue = "some_queue"
8
- end
9
-
10
- it "does not retry if no exception is raised" do
11
- mock(@raw_kestrel_client).get(@queue) { :mcguffin }
12
- lambda do
13
- @kestrel.get(@queue).should == :mcguffin
14
- end.should_not raise_error
15
- end
16
-
17
- ['get', 'set', 'delete'].each do |operation|
18
- it "retries DEFAULT_RETRY_COUNT times then fails" do
19
- mock(@raw_kestrel_client).send(operation, @queue) { raise Memcached::ServerIsMarkedDead }.
20
- times(Kestrel::Client::Retrying::DEFAULT_RETRY_COUNT + 1)
21
-
22
- lambda do
23
- @kestrel.send(operation, @queue)
24
- end.should raise_error(Memcached::ServerIsMarkedDead)
25
- end
26
-
27
- it "does not retry on non-connection related exceptions" do
28
- [Memcached::ABadKeyWasProvidedOrCharactersOutOfRange,
29
- Memcached::ActionQueued,
30
- Memcached::NoServersDefined].each do |ex|
31
-
32
- mock(@raw_kestrel_client).send(operation, @queue) { raise ex }
33
- lambda { @kestrel.send(operation, @queue) }.should raise_error(ex)
34
-
35
- end
36
- end
37
-
38
- it "does not retry when retry count is 0" do
39
- kestrel = Kestrel::Client::Retrying.new(@raw_kestrel_client, 0)
40
- mock(@raw_kestrel_client).send(operation, @queue) { raise Memcached::ServerIsMarkedDead }
41
- lambda { kestrel.send(operation, @queue) }.should raise_error(Memcached::ServerIsMarkedDead)
42
- end
43
-
44
- end
45
-
46
- end