siberite-client 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +3 -0
- data/Gemfile +4 -0
- data/LICENSE +14 -0
- data/README.md +63 -0
- data/Rakefile +18 -0
- data/lib/siberite-client.rb +1 -0
- data/lib/siberite.rb +9 -0
- data/lib/siberite/client.rb +202 -0
- data/lib/siberite/client/blocking.rb +36 -0
- data/lib/siberite/client/envelope.rb +25 -0
- data/lib/siberite/client/json.rb +16 -0
- data/lib/siberite/client/namespace.rb +29 -0
- data/lib/siberite/client/partitioning.rb +35 -0
- data/lib/siberite/client/proxy.rb +15 -0
- data/lib/siberite/client/stats_helper.rb +122 -0
- data/lib/siberite/client/transactional.rb +125 -0
- data/lib/siberite/client/unmarshal.rb +32 -0
- data/lib/siberite/client/version.rb +3 -0
- data/lib/siberite/config.rb +48 -0
- data/siberite-client.gemspec +35 -0
- data/spec/siberite/client/blocking_spec.rb +29 -0
- data/spec/siberite/client/envelope_spec.rb +45 -0
- data/spec/siberite/client/json_spec.rb +40 -0
- data/spec/siberite/client/namespace_spec.rb +54 -0
- data/spec/siberite/client/partitioning_spec.rb +33 -0
- data/spec/siberite/client/transactional_spec.rb +273 -0
- data/spec/siberite/client/unmarshal_spec.rb +68 -0
- data/spec/siberite/client_spec.rb +175 -0
- data/spec/siberite/config/siberite.yml +42 -0
- data/spec/siberite/config_spec.rb +72 -0
- data/spec/siberite_benchmark.rb +26 -0
- data/spec/spec.opts +3 -0
- data/spec/spec_helper.rb +33 -0
- metadata +205 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: e7848ea76b5512913552938cfac31d21fedce5b2
|
4
|
+
data.tar.gz: 7cbdf9d5fbed9c3aa10ccd8aa917d3f882f2f8a6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 1762a60bc54881a542068783b82ffe10bfbee2f5aeaba80831e0542e5365552bb9f3e6e21c72c77e8976ad90cf159ef0a1f94c4bad9598b701e010d263836eb6
|
7
|
+
data.tar.gz: c74d681cc09007725fbe9d6996e66ea4a6820c713266be633e2997cd227d04265e7c468d146b5dfdf996b2cd01abe2e7a866691d57fa010d3847f47c3d075f28
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Copyright 2015 Anton Bogdanovich
|
2
|
+
Copyright 2010-2014 Twitter, Inc.
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
## siberite-client: Talk to Siberite queue server from Ruby
|
2
|
+
|
3
|
+
siberite-client is a library that allows you to talk to a [Siberite](http://github.com/robey/siberite) queue server from ruby. As Siberite uses the memcache protocol, siberite-client is implemented as a wrapper around the memcached gem.
|
4
|
+
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
you will need to install memcached.gem, though rubygems should do this for you. just:
|
9
|
+
|
10
|
+
sudo gem install siberite-client
|
11
|
+
|
12
|
+
|
13
|
+
## Basic Usage
|
14
|
+
|
15
|
+
`Siberite::Client.new` takes a list of servers and an options hash. See the [rdoc for Memcached](http://blog.evanweaver.com/files/doc/fauna/memcached/classes/Memcached.html) for an explanation of what the various options do.
|
16
|
+
|
17
|
+
require 'siberite'
|
18
|
+
|
19
|
+
$queue = Siberite::Client.new('localhost:22133')
|
20
|
+
$queue.set('a_queue', 'foo')
|
21
|
+
$queue.get('a_queue') # => 'foo'
|
22
|
+
|
23
|
+
|
24
|
+
## Client Proxies
|
25
|
+
|
26
|
+
siberite-client comes with a number of decorators that change the behavior of the raw client.
|
27
|
+
|
28
|
+
$queue = Siberite::Client.new('localhost:22133')
|
29
|
+
$queue.get('empty_queue') # => nil
|
30
|
+
|
31
|
+
$queue = Siberite::Client::Blocking.new(Siberite::Client.new('localhost:22133'))
|
32
|
+
$queue.get('empty_queue') # does not return until it pulls something from the queue
|
33
|
+
|
34
|
+
|
35
|
+
## Configuration Management
|
36
|
+
|
37
|
+
Siberite::Config provides some tools for pulling queue config out of a YAML config file.
|
38
|
+
|
39
|
+
Siberite::Config.load 'path/to/siberite.yml'
|
40
|
+
Siberite::Config.environment = 'production' # defaults to development
|
41
|
+
|
42
|
+
$queue = Siberite::Config.new_client
|
43
|
+
|
44
|
+
This tells siberite-client to look for `path/to/siberite.yml`, and pull the client configuration out of
|
45
|
+
the 'production' key in that file. Sample config:
|
46
|
+
|
47
|
+
defaults: &defaults
|
48
|
+
distribution: :random
|
49
|
+
timeout: 2
|
50
|
+
connect_timeout: 1
|
51
|
+
|
52
|
+
production:
|
53
|
+
<<: *defaults
|
54
|
+
servers:
|
55
|
+
- siberite01.example.com:22133
|
56
|
+
- siberite02.example.com:22133
|
57
|
+
- siberite03.example.com:22133
|
58
|
+
|
59
|
+
development:
|
60
|
+
<<: *defaults
|
61
|
+
servers:
|
62
|
+
- localhost:22133
|
63
|
+
show_backtraces: true
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
ROOT_DIR = File.expand_path(File.dirname(__FILE__))
|
2
|
+
|
3
|
+
require 'rubygems' rescue nil
|
4
|
+
require 'rake'
|
5
|
+
require 'rspec/core/rake_task'
|
6
|
+
|
7
|
+
task :default => :spec
|
8
|
+
|
9
|
+
desc "Run all specs in spec directory."
|
10
|
+
RSpec::Core::RakeTask.new(:spec) do |t|
|
11
|
+
t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
|
12
|
+
end
|
13
|
+
|
14
|
+
desc "Run benchmarks"
|
15
|
+
RSpec::Core::RakeTask.new(:benchmark) do |t|
|
16
|
+
t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
|
17
|
+
t.pattern = 'spec/*_benchmark.rb'
|
18
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'siberite'
|
data/lib/siberite.rb
ADDED
@@ -0,0 +1,202 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
|
+
module Siberite
|
4
|
+
class Client
|
5
|
+
require 'siberite/client/stats_helper'
|
6
|
+
|
7
|
+
autoload :Proxy, 'siberite/client/proxy'
|
8
|
+
autoload :Envelope, 'siberite/client/envelope'
|
9
|
+
autoload :Blocking, 'siberite/client/blocking'
|
10
|
+
autoload :Partitioning, 'siberite/client/partitioning'
|
11
|
+
autoload :Unmarshal, 'siberite/client/unmarshal'
|
12
|
+
autoload :Namespace, 'siberite/client/namespace'
|
13
|
+
autoload :Json, 'siberite/client/json'
|
14
|
+
autoload :Transactional, "siberite/client/transactional"
|
15
|
+
|
16
|
+
SIBERITE_OPTIONS = [:gets_per_server, :exception_retry_limit, :get_timeout_ms].freeze
|
17
|
+
|
18
|
+
DEFAULT_OPTIONS = {
|
19
|
+
:retry_timeout => 0,
|
20
|
+
:exception_retry_limit => 5,
|
21
|
+
:timeout => 0.25,
|
22
|
+
:gets_per_server => 100,
|
23
|
+
:get_timeout_ms => 10
|
24
|
+
}.freeze
|
25
|
+
|
26
|
+
# Exceptions which are connection failures we retry after
|
27
|
+
RECOVERABLE_ERRORS = [
|
28
|
+
Memcached::ServerIsMarkedDead,
|
29
|
+
Memcached::ATimeoutOccurred,
|
30
|
+
Memcached::ConnectionBindFailure,
|
31
|
+
Memcached::ConnectionFailure,
|
32
|
+
Memcached::ConnectionSocketCreateFailure,
|
33
|
+
Memcached::Failure,
|
34
|
+
Memcached::MemoryAllocationFailure,
|
35
|
+
Memcached::ReadFailure,
|
36
|
+
Memcached::ServerError,
|
37
|
+
Memcached::SystemError,
|
38
|
+
Memcached::UnknownReadFailure,
|
39
|
+
Memcached::WriteFailure,
|
40
|
+
Memcached::NotFound
|
41
|
+
]
|
42
|
+
|
43
|
+
extend Forwardable
|
44
|
+
include StatsHelper
|
45
|
+
|
46
|
+
attr_accessor :servers, :options
|
47
|
+
attr_reader :current_queue, :siberite_options, :current_server
|
48
|
+
|
49
|
+
def_delegators :@write_client, :add, :append, :cas, :decr, :incr, :get_orig, :prepend
|
50
|
+
|
51
|
+
def initialize(*servers)
|
52
|
+
opts = servers.last.is_a?(Hash) ? servers.pop : {}
|
53
|
+
opts = DEFAULT_OPTIONS.merge(opts)
|
54
|
+
|
55
|
+
@siberite_options = extract_siberite_options!(opts)
|
56
|
+
@default_get_timeout = siberite_options[:get_timeout_ms]
|
57
|
+
@gets_per_server = siberite_options[:gets_per_server]
|
58
|
+
@exception_retry_limit = siberite_options[:exception_retry_limit]
|
59
|
+
@counter = 0
|
60
|
+
@shuffle = true
|
61
|
+
|
62
|
+
# we handle our own retries so that we can apply different
|
63
|
+
# policies to sets and gets, so set memcached limit to 0
|
64
|
+
opts[:exception_retry_limit] = 0
|
65
|
+
opts[:distribution] = :random # force random distribution
|
66
|
+
|
67
|
+
self.servers = Array(servers).flatten.compact
|
68
|
+
self.options = opts
|
69
|
+
|
70
|
+
@server_count = self.servers.size # Minor optimization.
|
71
|
+
@read_client = Memcached.new(self.servers[rand(@server_count)], opts)
|
72
|
+
@write_client = Memcached.new(self.servers[rand(@server_count)], opts)
|
73
|
+
end
|
74
|
+
|
75
|
+
def delete(key, expiry=0)
|
76
|
+
with_retries { @write_client.delete key }
|
77
|
+
rescue Memcached::NotFound, Memcached::ServerEnd
|
78
|
+
end
|
79
|
+
|
80
|
+
def set(key, value, ttl=0, raw=false)
|
81
|
+
with_retries { @write_client.set key, value, ttl, !raw }
|
82
|
+
true
|
83
|
+
rescue Memcached::NotStored
|
84
|
+
false
|
85
|
+
end
|
86
|
+
|
87
|
+
# This provides the necessary semantic to support transactionality
|
88
|
+
# in the Transactional client. It temporarily disables server
|
89
|
+
# shuffling to allow the client to close any open transactions on
|
90
|
+
# the current server before jumping.
|
91
|
+
#
|
92
|
+
def get_from_last(*args)
|
93
|
+
@shuffle = false
|
94
|
+
get *args
|
95
|
+
ensure
|
96
|
+
@shuffle = true
|
97
|
+
end
|
98
|
+
|
99
|
+
# ==== Parameters
|
100
|
+
# key<String>:: Queue name
|
101
|
+
# opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
|
102
|
+
# allows collision-avoiding options support.
|
103
|
+
#
|
104
|
+
# ==== Options (opts)
|
105
|
+
# :open<Boolean>:: Begins a transactional read.
|
106
|
+
# :close<Boolean>:: Ends a transactional read.
|
107
|
+
# :abort<Boolean>:: Cancels an existing transactional read
|
108
|
+
# :peek<Boolean>:: Return the head of the queue, without removal
|
109
|
+
# :timeout<Integer>:: Milliseconds to block for a new item
|
110
|
+
# :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
|
111
|
+
# style" second argument.
|
112
|
+
#
|
113
|
+
def get(key, opts = {})
|
114
|
+
raw = opts[:raw] || false
|
115
|
+
commands = extract_queue_commands(opts)
|
116
|
+
|
117
|
+
val =
|
118
|
+
begin
|
119
|
+
shuffle_if_necessary! key
|
120
|
+
@read_client.get key + commands, !raw
|
121
|
+
rescue *RECOVERABLE_ERRORS
|
122
|
+
# we can't tell the difference between a server being down
|
123
|
+
# and an empty queue, so just return nil. our sticky server
|
124
|
+
# logic should eliminate piling on down servers
|
125
|
+
nil
|
126
|
+
end
|
127
|
+
|
128
|
+
# nil result without :close and :abort, force next get to jump from
|
129
|
+
# current server
|
130
|
+
if !val && @shuffle && !opts[:close] && !opts[:abort]
|
131
|
+
@counter = @gets_per_server
|
132
|
+
end
|
133
|
+
|
134
|
+
val
|
135
|
+
end
|
136
|
+
|
137
|
+
def flush(queue)
|
138
|
+
count = 0
|
139
|
+
while sizeof(queue) > 0
|
140
|
+
count += 1 while get queue, :raw => true
|
141
|
+
end
|
142
|
+
count
|
143
|
+
end
|
144
|
+
|
145
|
+
def peek(queue)
|
146
|
+
get queue, :peek => true
|
147
|
+
end
|
148
|
+
|
149
|
+
private
|
150
|
+
|
151
|
+
def extract_siberite_options!(opts)
|
152
|
+
siberite_opts, memcache_opts = opts.inject([{}, {}]) do |(siberite, memcache), (key, opt)|
|
153
|
+
(SIBERITE_OPTIONS.include?(key) ? siberite : memcache)[key] = opt
|
154
|
+
[siberite, memcache]
|
155
|
+
end
|
156
|
+
opts.replace(memcache_opts)
|
157
|
+
siberite_opts
|
158
|
+
end
|
159
|
+
|
160
|
+
def shuffle_if_necessary!(key)
|
161
|
+
return unless @server_count > 1
|
162
|
+
# Don't reset servers on the first request:
|
163
|
+
# i.e. @counter == 0 && @current_queue == nil
|
164
|
+
if @shuffle &&
|
165
|
+
(@counter > 0 && key != @current_queue) ||
|
166
|
+
@counter >= @gets_per_server
|
167
|
+
@counter = 0
|
168
|
+
@current_queue = key
|
169
|
+
@read_client.reset(servers[rand(@server_count)])
|
170
|
+
else
|
171
|
+
@counter +=1
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def extract_queue_commands(opts)
|
176
|
+
commands = [:open, :close, :abort, :peek].select do |key|
|
177
|
+
opts[key]
|
178
|
+
end
|
179
|
+
|
180
|
+
if timeout = (opts[:timeout] || @default_get_timeout)
|
181
|
+
commands << "t=#{timeout}"
|
182
|
+
end
|
183
|
+
|
184
|
+
commands.map { |c| "/#{c}" }.join('')
|
185
|
+
end
|
186
|
+
|
187
|
+
def with_retries #:nodoc:
|
188
|
+
yield
|
189
|
+
rescue *RECOVERABLE_ERRORS
|
190
|
+
tries ||= @exception_retry_limit + 1
|
191
|
+
tries -= 1
|
192
|
+
@write_client.reset(servers[rand(@server_count)])
|
193
|
+
|
194
|
+
if tries > 0
|
195
|
+
retry
|
196
|
+
else
|
197
|
+
raise
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
end
|
202
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Siberite
|
2
|
+
class Client
|
3
|
+
class Blocking < Proxy
|
4
|
+
|
5
|
+
# random backoff sleeping
|
6
|
+
|
7
|
+
SLEEP_TIMES = [[0] * 1, [0.01] * 2, [0.1] * 2, [0.5] * 2, [1.0] * 1].flatten
|
8
|
+
|
9
|
+
def get(*args)
|
10
|
+
count = 0
|
11
|
+
|
12
|
+
while count += 1
|
13
|
+
|
14
|
+
if response = client.get(*args)
|
15
|
+
return response
|
16
|
+
end
|
17
|
+
|
18
|
+
sleep_for_count(count)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def get_without_blocking(*args)
|
23
|
+
client.get(*args)
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def sleep_for_count(count)
|
29
|
+
base = SLEEP_TIMES[count] || SLEEP_TIMES.last
|
30
|
+
|
31
|
+
time = ((rand * base) + base) / 2
|
32
|
+
sleep time if time > 0
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Siberite
|
2
|
+
class Client
|
3
|
+
class Envelope < Proxy
|
4
|
+
attr_accessor :envelope_class
|
5
|
+
|
6
|
+
def initialize(envelope_class, client)
|
7
|
+
@envelope_class = envelope_class
|
8
|
+
super(client)
|
9
|
+
end
|
10
|
+
|
11
|
+
def get(*args)
|
12
|
+
response = client.get(*args)
|
13
|
+
if response.respond_to?(:unwrap)
|
14
|
+
response.unwrap
|
15
|
+
else
|
16
|
+
response
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def set(key, value, *args)
|
21
|
+
client.set(key, envelope_class.new(value), *args)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Siberite
|
4
|
+
class Client
|
5
|
+
class Json < Proxy
|
6
|
+
def get(*args)
|
7
|
+
response = client.get(*args)
|
8
|
+
if response.is_a?(String)
|
9
|
+
HashWithIndifferentAccess.new(JSON.parse(response)) rescue response
|
10
|
+
else
|
11
|
+
response
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module Siberite
|
2
|
+
class Client
|
3
|
+
class Namespace < Proxy
|
4
|
+
def initialize(namespace, client)
|
5
|
+
@namespace = namespace
|
6
|
+
@matcher = /\A#{Regexp.escape(@namespace)}:(.+)/
|
7
|
+
super(client)
|
8
|
+
end
|
9
|
+
|
10
|
+
%w(set get delete flush stat).each do |method|
|
11
|
+
class_eval "def #{method}(key, *args); client.#{method}(namespace(key), *args) end", __FILE__, __LINE__
|
12
|
+
end
|
13
|
+
|
14
|
+
def available_queues
|
15
|
+
client.available_queues.map {|q| in_namespace(q) }.compact
|
16
|
+
end
|
17
|
+
|
18
|
+
def in_namespace(key)
|
19
|
+
if match = @matcher.match(key)
|
20
|
+
match[1]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def namespace(key)
|
25
|
+
"#{@namespace}:#{key}"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Siberite
|
2
|
+
class Client
|
3
|
+
class Partitioning < Proxy
|
4
|
+
|
5
|
+
def initialize(client_map)
|
6
|
+
@clients = client_map.inject({}) do |clients, (keys, client)|
|
7
|
+
Array(keys).inject(clients) do |_, key|
|
8
|
+
clients.update(key => client)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def clients
|
14
|
+
@clients.values.uniq
|
15
|
+
end
|
16
|
+
|
17
|
+
def default_client
|
18
|
+
@clients[:default]
|
19
|
+
end
|
20
|
+
alias client default_client
|
21
|
+
|
22
|
+
%w(set get delete flush stat).each do |method|
|
23
|
+
class_eval "def #{method}(key, *args); client_for(key).#{method}(key, *args) end", __FILE__, __LINE__
|
24
|
+
end
|
25
|
+
|
26
|
+
def stats
|
27
|
+
merge_stats(clients.map {|c| c.stats })
|
28
|
+
end
|
29
|
+
|
30
|
+
def client_for(key)
|
31
|
+
@clients[key.to_s.split('/', 2).first] || default_client
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|