siberite-client 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: e7848ea76b5512913552938cfac31d21fedce5b2
4
+ data.tar.gz: 7cbdf9d5fbed9c3aa10ccd8aa917d3f882f2f8a6
5
+ SHA512:
6
+ metadata.gz: 1762a60bc54881a542068783b82ffe10bfbee2f5aeaba80831e0542e5365552bb9f3e6e21c72c77e8976ad90cf159ef0a1f94c4bad9598b701e010d263836eb6
7
+ data.tar.gz: c74d681cc09007725fbe9d6996e66ea4a6820c713266be633e2997cd227d04265e7c468d146b5dfdf996b2cd01abe2e7a866691d57fa010d3847f47c3d075f28
@@ -0,0 +1,3 @@
1
+ pkg/*
2
+ Gemfile.lock
3
+ *.gem
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in ngrok-tunnel.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,14 @@
1
+ Copyright 2015 Anton Bogdanovich
2
+ Copyright 2010-2014 Twitter, Inc.
3
+
4
+ Licensed under the Apache License, Version 2.0 (the "License");
5
+ you may not use this file except in compliance with the License.
6
+ You may obtain a copy of the License at
7
+
8
+ http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ Unless required by applicable law or agreed to in writing, software
11
+ distributed under the License is distributed on an "AS IS" BASIS,
12
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ See the License for the specific language governing permissions and
14
+ limitations under the License.
@@ -0,0 +1,63 @@
1
+ ## siberite-client: Talk to Siberite queue server from Ruby
2
+
3
+ siberite-client is a library that allows you to talk to a [Siberite](http://github.com/robey/siberite) queue server from ruby. As Siberite uses the memcache protocol, siberite-client is implemented as a wrapper around the memcached gem.
4
+
5
+
6
+ ## Installation
7
+
8
+ you will need to install memcached.gem, though rubygems should do this for you. just:
9
+
10
+ sudo gem install siberite-client
11
+
12
+
13
+ ## Basic Usage
14
+
15
+ `Siberite::Client.new` takes a list of servers and an options hash. See the [rdoc for Memcached](http://blog.evanweaver.com/files/doc/fauna/memcached/classes/Memcached.html) for an explanation of what the various options do.
16
+
17
+ require 'siberite'
18
+
19
+ $queue = Siberite::Client.new('localhost:22133')
20
+ $queue.set('a_queue', 'foo')
21
+ $queue.get('a_queue') # => 'foo'
22
+
23
+
24
+ ## Client Proxies
25
+
26
+ siberite-client comes with a number of decorators that change the behavior of the raw client.
27
+
28
+ $queue = Siberite::Client.new('localhost:22133')
29
+ $queue.get('empty_queue') # => nil
30
+
31
+ $queue = Siberite::Client::Blocking.new(Siberite::Client.new('localhost:22133'))
32
+ $queue.get('empty_queue') # does not return until it pulls something from the queue
33
+
34
+
35
+ ## Configuration Management
36
+
37
+ Siberite::Config provides some tools for pulling queue config out of a YAML config file.
38
+
39
+ Siberite::Config.load 'path/to/siberite.yml'
40
+ Siberite::Config.environment = 'production' # defaults to development
41
+
42
+ $queue = Siberite::Config.new_client
43
+
44
+ This tells siberite-client to look for `path/to/siberite.yml`, and pull the client configuration out of
45
+ the 'production' key in that file. Sample config:
46
+
47
+ defaults: &defaults
48
+ distribution: :random
49
+ timeout: 2
50
+ connect_timeout: 1
51
+
52
+ production:
53
+ <<: *defaults
54
+ servers:
55
+ - siberite01.example.com:22133
56
+ - siberite02.example.com:22133
57
+ - siberite03.example.com:22133
58
+
59
+ development:
60
+ <<: *defaults
61
+ servers:
62
+ - localhost:22133
63
+ show_backtraces: true
@@ -0,0 +1,18 @@
1
+ ROOT_DIR = File.expand_path(File.dirname(__FILE__))
2
+
3
+ require 'rubygems' rescue nil
4
+ require 'rake'
5
+ require 'rspec/core/rake_task'
6
+
7
+ task :default => :spec
8
+
9
+ desc "Run all specs in spec directory."
10
+ RSpec::Core::RakeTask.new(:spec) do |t|
11
+ t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
12
+ end
13
+
14
+ desc "Run benchmarks"
15
+ RSpec::Core::RakeTask.new(:benchmark) do |t|
16
+ t.rspec_opts = ['--options', "\"#{ROOT_DIR}/spec/spec.opts\""]
17
+ t.pattern = 'spec/*_benchmark.rb'
18
+ end
@@ -0,0 +1 @@
1
+ require 'siberite'
@@ -0,0 +1,9 @@
1
+ require 'yaml'
2
+ require 'socket'
3
+ require 'memcached'
4
+
5
+ require 'siberite/client'
6
+
7
+ module Siberite
8
+ autoload :Config, 'siberite/config'
9
+ end
@@ -0,0 +1,202 @@
1
+ require 'forwardable'
2
+
3
+ module Siberite
4
+ class Client
5
+ require 'siberite/client/stats_helper'
6
+
7
+ autoload :Proxy, 'siberite/client/proxy'
8
+ autoload :Envelope, 'siberite/client/envelope'
9
+ autoload :Blocking, 'siberite/client/blocking'
10
+ autoload :Partitioning, 'siberite/client/partitioning'
11
+ autoload :Unmarshal, 'siberite/client/unmarshal'
12
+ autoload :Namespace, 'siberite/client/namespace'
13
+ autoload :Json, 'siberite/client/json'
14
+ autoload :Transactional, "siberite/client/transactional"
15
+
16
+ SIBERITE_OPTIONS = [:gets_per_server, :exception_retry_limit, :get_timeout_ms].freeze
17
+
18
+ DEFAULT_OPTIONS = {
19
+ :retry_timeout => 0,
20
+ :exception_retry_limit => 5,
21
+ :timeout => 0.25,
22
+ :gets_per_server => 100,
23
+ :get_timeout_ms => 10
24
+ }.freeze
25
+
26
+ # Exceptions which are connection failures we retry after
27
+ RECOVERABLE_ERRORS = [
28
+ Memcached::ServerIsMarkedDead,
29
+ Memcached::ATimeoutOccurred,
30
+ Memcached::ConnectionBindFailure,
31
+ Memcached::ConnectionFailure,
32
+ Memcached::ConnectionSocketCreateFailure,
33
+ Memcached::Failure,
34
+ Memcached::MemoryAllocationFailure,
35
+ Memcached::ReadFailure,
36
+ Memcached::ServerError,
37
+ Memcached::SystemError,
38
+ Memcached::UnknownReadFailure,
39
+ Memcached::WriteFailure,
40
+ Memcached::NotFound
41
+ ]
42
+
43
+ extend Forwardable
44
+ include StatsHelper
45
+
46
+ attr_accessor :servers, :options
47
+ attr_reader :current_queue, :siberite_options, :current_server
48
+
49
+ def_delegators :@write_client, :add, :append, :cas, :decr, :incr, :get_orig, :prepend
50
+
51
+ def initialize(*servers)
52
+ opts = servers.last.is_a?(Hash) ? servers.pop : {}
53
+ opts = DEFAULT_OPTIONS.merge(opts)
54
+
55
+ @siberite_options = extract_siberite_options!(opts)
56
+ @default_get_timeout = siberite_options[:get_timeout_ms]
57
+ @gets_per_server = siberite_options[:gets_per_server]
58
+ @exception_retry_limit = siberite_options[:exception_retry_limit]
59
+ @counter = 0
60
+ @shuffle = true
61
+
62
+ # we handle our own retries so that we can apply different
63
+ # policies to sets and gets, so set memcached limit to 0
64
+ opts[:exception_retry_limit] = 0
65
+ opts[:distribution] = :random # force random distribution
66
+
67
+ self.servers = Array(servers).flatten.compact
68
+ self.options = opts
69
+
70
+ @server_count = self.servers.size # Minor optimization.
71
+ @read_client = Memcached.new(self.servers[rand(@server_count)], opts)
72
+ @write_client = Memcached.new(self.servers[rand(@server_count)], opts)
73
+ end
74
+
75
+ def delete(key, expiry=0)
76
+ with_retries { @write_client.delete key }
77
+ rescue Memcached::NotFound, Memcached::ServerEnd
78
+ end
79
+
80
+ def set(key, value, ttl=0, raw=false)
81
+ with_retries { @write_client.set key, value, ttl, !raw }
82
+ true
83
+ rescue Memcached::NotStored
84
+ false
85
+ end
86
+
87
+ # This provides the necessary semantic to support transactionality
88
+ # in the Transactional client. It temporarily disables server
89
+ # shuffling to allow the client to close any open transactions on
90
+ # the current server before jumping.
91
+ #
92
+ def get_from_last(*args)
93
+ @shuffle = false
94
+ get *args
95
+ ensure
96
+ @shuffle = true
97
+ end
98
+
99
+ # ==== Parameters
100
+ # key<String>:: Queue name
101
+ # opts<Boolean,Hash>:: True/false toggles Marshalling. A Hash
102
+ # allows collision-avoiding options support.
103
+ #
104
+ # ==== Options (opts)
105
+ # :open<Boolean>:: Begins a transactional read.
106
+ # :close<Boolean>:: Ends a transactional read.
107
+ # :abort<Boolean>:: Cancels an existing transactional read
108
+ # :peek<Boolean>:: Return the head of the queue, without removal
109
+ # :timeout<Integer>:: Milliseconds to block for a new item
110
+ # :raw<Boolean>:: Toggles Marshalling. Equivalent to the "old
111
+ # style" second argument.
112
+ #
113
+ def get(key, opts = {})
114
+ raw = opts[:raw] || false
115
+ commands = extract_queue_commands(opts)
116
+
117
+ val =
118
+ begin
119
+ shuffle_if_necessary! key
120
+ @read_client.get key + commands, !raw
121
+ rescue *RECOVERABLE_ERRORS
122
+ # we can't tell the difference between a server being down
123
+ # and an empty queue, so just return nil. our sticky server
124
+ # logic should eliminate piling on down servers
125
+ nil
126
+ end
127
+
128
+ # nil result without :close and :abort, force next get to jump from
129
+ # current server
130
+ if !val && @shuffle && !opts[:close] && !opts[:abort]
131
+ @counter = @gets_per_server
132
+ end
133
+
134
+ val
135
+ end
136
+
137
+ def flush(queue)
138
+ count = 0
139
+ while sizeof(queue) > 0
140
+ count += 1 while get queue, :raw => true
141
+ end
142
+ count
143
+ end
144
+
145
+ def peek(queue)
146
+ get queue, :peek => true
147
+ end
148
+
149
+ private
150
+
151
+ def extract_siberite_options!(opts)
152
+ siberite_opts, memcache_opts = opts.inject([{}, {}]) do |(siberite, memcache), (key, opt)|
153
+ (SIBERITE_OPTIONS.include?(key) ? siberite : memcache)[key] = opt
154
+ [siberite, memcache]
155
+ end
156
+ opts.replace(memcache_opts)
157
+ siberite_opts
158
+ end
159
+
160
+ def shuffle_if_necessary!(key)
161
+ return unless @server_count > 1
162
+ # Don't reset servers on the first request:
163
+ # i.e. @counter == 0 && @current_queue == nil
164
+ if @shuffle &&
165
+ (@counter > 0 && key != @current_queue) ||
166
+ @counter >= @gets_per_server
167
+ @counter = 0
168
+ @current_queue = key
169
+ @read_client.reset(servers[rand(@server_count)])
170
+ else
171
+ @counter +=1
172
+ end
173
+ end
174
+
175
+ def extract_queue_commands(opts)
176
+ commands = [:open, :close, :abort, :peek].select do |key|
177
+ opts[key]
178
+ end
179
+
180
+ if timeout = (opts[:timeout] || @default_get_timeout)
181
+ commands << "t=#{timeout}"
182
+ end
183
+
184
+ commands.map { |c| "/#{c}" }.join('')
185
+ end
186
+
187
+ def with_retries #:nodoc:
188
+ yield
189
+ rescue *RECOVERABLE_ERRORS
190
+ tries ||= @exception_retry_limit + 1
191
+ tries -= 1
192
+ @write_client.reset(servers[rand(@server_count)])
193
+
194
+ if tries > 0
195
+ retry
196
+ else
197
+ raise
198
+ end
199
+ end
200
+
201
+ end
202
+ end
@@ -0,0 +1,36 @@
1
+ module Siberite
2
+ class Client
3
+ class Blocking < Proxy
4
+
5
+ # random backoff sleeping
6
+
7
+ SLEEP_TIMES = [[0] * 1, [0.01] * 2, [0.1] * 2, [0.5] * 2, [1.0] * 1].flatten
8
+
9
+ def get(*args)
10
+ count = 0
11
+
12
+ while count += 1
13
+
14
+ if response = client.get(*args)
15
+ return response
16
+ end
17
+
18
+ sleep_for_count(count)
19
+ end
20
+ end
21
+
22
+ def get_without_blocking(*args)
23
+ client.get(*args)
24
+ end
25
+
26
+ private
27
+
28
+ def sleep_for_count(count)
29
+ base = SLEEP_TIMES[count] || SLEEP_TIMES.last
30
+
31
+ time = ((rand * base) + base) / 2
32
+ sleep time if time > 0
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,25 @@
1
+ module Siberite
2
+ class Client
3
+ class Envelope < Proxy
4
+ attr_accessor :envelope_class
5
+
6
+ def initialize(envelope_class, client)
7
+ @envelope_class = envelope_class
8
+ super(client)
9
+ end
10
+
11
+ def get(*args)
12
+ response = client.get(*args)
13
+ if response.respond_to?(:unwrap)
14
+ response.unwrap
15
+ else
16
+ response
17
+ end
18
+ end
19
+
20
+ def set(key, value, *args)
21
+ client.set(key, envelope_class.new(value), *args)
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,16 @@
1
+ require 'json'
2
+
3
+ module Siberite
4
+ class Client
5
+ class Json < Proxy
6
+ def get(*args)
7
+ response = client.get(*args)
8
+ if response.is_a?(String)
9
+ HashWithIndifferentAccess.new(JSON.parse(response)) rescue response
10
+ else
11
+ response
12
+ end
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,29 @@
1
+ module Siberite
2
+ class Client
3
+ class Namespace < Proxy
4
+ def initialize(namespace, client)
5
+ @namespace = namespace
6
+ @matcher = /\A#{Regexp.escape(@namespace)}:(.+)/
7
+ super(client)
8
+ end
9
+
10
+ %w(set get delete flush stat).each do |method|
11
+ class_eval "def #{method}(key, *args); client.#{method}(namespace(key), *args) end", __FILE__, __LINE__
12
+ end
13
+
14
+ def available_queues
15
+ client.available_queues.map {|q| in_namespace(q) }.compact
16
+ end
17
+
18
+ def in_namespace(key)
19
+ if match = @matcher.match(key)
20
+ match[1]
21
+ end
22
+ end
23
+
24
+ def namespace(key)
25
+ "#{@namespace}:#{key}"
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,35 @@
1
+ module Siberite
2
+ class Client
3
+ class Partitioning < Proxy
4
+
5
+ def initialize(client_map)
6
+ @clients = client_map.inject({}) do |clients, (keys, client)|
7
+ Array(keys).inject(clients) do |_, key|
8
+ clients.update(key => client)
9
+ end
10
+ end
11
+ end
12
+
13
+ def clients
14
+ @clients.values.uniq
15
+ end
16
+
17
+ def default_client
18
+ @clients[:default]
19
+ end
20
+ alias client default_client
21
+
22
+ %w(set get delete flush stat).each do |method|
23
+ class_eval "def #{method}(key, *args); client_for(key).#{method}(key, *args) end", __FILE__, __LINE__
24
+ end
25
+
26
+ def stats
27
+ merge_stats(clients.map {|c| c.stats })
28
+ end
29
+
30
+ def client_for(key)
31
+ @clients[key.to_s.split('/', 2).first] || default_client
32
+ end
33
+ end
34
+ end
35
+ end