beaneater 0.3.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -1
- data/README.md +37 -19
- data/examples/demo.rb +1 -2
- data/lib/beaneater.rb +56 -3
- data/lib/beaneater/configuration.rb +3 -1
- data/lib/beaneater/connection.rb +123 -26
- data/lib/beaneater/errors.rb +1 -1
- data/lib/beaneater/job/collection.rb +57 -36
- data/lib/beaneater/job/record.rb +27 -27
- data/lib/beaneater/stats.rb +28 -5
- data/lib/beaneater/stats/fast_struct.rb +1 -1
- data/lib/beaneater/stats/stat_struct.rb +8 -2
- data/lib/beaneater/tube/collection.rb +67 -26
- data/lib/beaneater/tube/record.rb +38 -24
- data/lib/beaneater/version.rb +3 -3
- data/test/beaneater_test.rb +1 -7
- data/test/connection_test.rb +37 -1
- data/test/job_test.rb +3 -7
- data/test/jobs_test.rb +12 -40
- data/test/prompt_regexp_test.rb +4 -3
- data/test/stat_struct_test.rb +10 -0
- data/test/stats_test.rb +7 -7
- data/test/test_helper.rb +30 -6
- data/test/tube_test.rb +9 -20
- data/test/tubes_test.rb +59 -54
- metadata +2 -8
- data/lib/beaneater/pool.rb +0 -166
- data/lib/beaneater/pool_command.rb +0 -79
- data/test/pool_command_test.rb +0 -90
- data/test/pool_test.rb +0 -185
@@ -1,79 +0,0 @@
|
|
1
|
-
require 'set'
|
2
|
-
|
3
|
-
module Beaneater
|
4
|
-
# Represents collection of pool related commands.
|
5
|
-
class PoolCommand
|
6
|
-
# @!attribute pool
|
7
|
-
# @return [Beaneater::Pool] returns Pool object
|
8
|
-
attr_reader :pool
|
9
|
-
|
10
|
-
# Initialize new connection
|
11
|
-
#
|
12
|
-
# @param [Beaneater::Pool] pool Pool object
|
13
|
-
def initialize(pool)
|
14
|
-
@pool = pool
|
15
|
-
end
|
16
|
-
|
17
|
-
# Delegate to Pool#transmit_to_all and if needed will merge responses from beanstalkd.
|
18
|
-
#
|
19
|
-
# @param [String] body Beanstalkd command
|
20
|
-
# @param [Hash{String => String, Boolean}] options socket connections options
|
21
|
-
# @option options [Boolean] merge Ask for merging responses or not
|
22
|
-
# @param [Proc] block Block passed in socket connection object
|
23
|
-
# @example
|
24
|
-
# @pool.transmit_to_all("stats")
|
25
|
-
#
|
26
|
-
def transmit_to_all(body, options={}, &block)
|
27
|
-
merge = options.delete(:merge)
|
28
|
-
res = pool.transmit_to_all(body, options, &block)
|
29
|
-
first = res.find { |r| r && r[:status] }
|
30
|
-
if first && merge
|
31
|
-
res = { :status => first[:status], :body => sum_items(res.map { |r| r[:body] }) }
|
32
|
-
end
|
33
|
-
res
|
34
|
-
end
|
35
|
-
|
36
|
-
# Delegate missing methods to pool
|
37
|
-
# @api public
|
38
|
-
def method_missing(name, *args, &block)
|
39
|
-
if pool.respond_to?(name)
|
40
|
-
pool.send(name, *args, &block)
|
41
|
-
else # not a known pool command
|
42
|
-
super
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
protected
|
47
|
-
|
48
|
-
# Selects items from collection and then merges the individual values
|
49
|
-
# Supports array of hashes or array of arrays
|
50
|
-
#
|
51
|
-
# @param [Array<Hash, Array>] hs Collection of responses returned from beanstalkd
|
52
|
-
# @return [Hash{Symbol => String}] Merged responses combining values from all the hash bodies
|
53
|
-
# @example
|
54
|
-
# self.sum_items([{ :foo => 1, :bar => 5 }, { :foo => 2, :bar => 3 }])
|
55
|
-
# => { :foo => 3, :bar => 8 }
|
56
|
-
# self.sum_items([['foo', 'bar'], ['foo', 'bar', 'baz']])
|
57
|
-
# => ['foo', 'bar', 'baz']
|
58
|
-
#
|
59
|
-
def sum_items(items)
|
60
|
-
if items.first.is_a?(Hash)
|
61
|
-
items.select { |h| h.is_a?(Hash) }.
|
62
|
-
inject({}) { |a,b| a.merge(b) { |k,o,n| combine_stats(k, o, n) } }
|
63
|
-
elsif items.first.is_a?(Array)
|
64
|
-
items.flatten.uniq
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
# Combine two values for given key
|
69
|
-
#
|
70
|
-
# @param [String] k key name within response hash
|
71
|
-
# @return [Set,Integer] combined value for stat
|
72
|
-
# @example
|
73
|
-
# self.combine_stats('total_connections', 4, 5) # => 9
|
74
|
-
#
|
75
|
-
def combine_stats(k, a, b)
|
76
|
-
['name', 'version', 'pid'].include?(k) ? Set[a] + Set[b] : a + b
|
77
|
-
end
|
78
|
-
end # PoolCommand
|
79
|
-
end # Beaneater
|
data/test/pool_command_test.rb
DELETED
@@ -1,90 +0,0 @@
|
|
1
|
-
# test/pool_command_test.rb
|
2
|
-
|
3
|
-
require File.expand_path('../test_helper', __FILE__)
|
4
|
-
|
5
|
-
describe Beaneater::PoolCommand do
|
6
|
-
|
7
|
-
describe 'for #new' do
|
8
|
-
before do
|
9
|
-
@pool = stub
|
10
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
11
|
-
end
|
12
|
-
|
13
|
-
it "should store pool" do
|
14
|
-
assert_equal @pool, @command.pool
|
15
|
-
end
|
16
|
-
end #new
|
17
|
-
|
18
|
-
describe 'for #transmit_to_all' do
|
19
|
-
describe 'for regular command' do
|
20
|
-
before do
|
21
|
-
@pool = stub(:transmit_to_all => [{ :body => "foo", :status => "OK" }])
|
22
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
23
|
-
end
|
24
|
-
|
25
|
-
it "can run regular command" do
|
26
|
-
res = @command.transmit_to_all("foo")
|
27
|
-
assert_equal "OK", res[0][:status]
|
28
|
-
assert_equal "foo", res[0][:body]
|
29
|
-
end
|
30
|
-
end # regular command
|
31
|
-
|
32
|
-
describe 'for merge command with hashes' do
|
33
|
-
before do
|
34
|
-
@pool = stub(:transmit_to_all => [
|
35
|
-
{ :body => { 'x' => 1, 'version' => 1.1 }, :status => "OK"},
|
36
|
-
{ :body => { 'x' => 3,'version' => 1.2 }, :status => "OK" }
|
37
|
-
])
|
38
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
39
|
-
end
|
40
|
-
|
41
|
-
it "can run merge command " do
|
42
|
-
cmd = @command.transmit_to_all("bar", :merge => true)
|
43
|
-
assert_equal "OK", cmd[:status]
|
44
|
-
assert_equal 4, cmd[:body]['x']
|
45
|
-
assert_equal Set[1.1, 1.2], cmd[:body]['version']
|
46
|
-
end
|
47
|
-
end # merge command
|
48
|
-
|
49
|
-
describe 'for merge command with arrays' do
|
50
|
-
before do
|
51
|
-
@pool = stub(:transmit_to_all => [
|
52
|
-
{ :body => ['foo', 'bar'], :status => "OK"},
|
53
|
-
{ :body => ['foo', 'bar', 'baz'], :status => "OK" }
|
54
|
-
])
|
55
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
56
|
-
end
|
57
|
-
|
58
|
-
it "can run merge command " do
|
59
|
-
cmd = @command.transmit_to_all("bar", :merge => true)
|
60
|
-
assert_equal "OK", cmd[:status]
|
61
|
-
assert_equal ['foo', 'bar', 'baz'].sort, cmd[:body].sort
|
62
|
-
end
|
63
|
-
end # merge command
|
64
|
-
end # transmit_to_all
|
65
|
-
|
66
|
-
describe 'for #method_missing' do
|
67
|
-
describe '#transmit_to_rand' do
|
68
|
-
before do
|
69
|
-
@pool = stub
|
70
|
-
@pool.expects(:transmit_to_rand).with('foo').returns('OK')
|
71
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
72
|
-
end
|
73
|
-
|
74
|
-
it 'delegates to connection' do
|
75
|
-
assert_equal 'OK', @command.transmit_to_rand('foo')
|
76
|
-
end
|
77
|
-
end # transmit_to_rand
|
78
|
-
|
79
|
-
describe 'invalid method' do
|
80
|
-
before do
|
81
|
-
@pool = stub
|
82
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
83
|
-
end
|
84
|
-
|
85
|
-
it 'raises no method error' do
|
86
|
-
assert_raises(NoMethodError) { @command.foo('foo') }
|
87
|
-
end
|
88
|
-
end # invalid method
|
89
|
-
end # method_missing
|
90
|
-
end # Beaneater::PoolCommand
|
data/test/pool_test.rb
DELETED
@@ -1,185 +0,0 @@
|
|
1
|
-
# test/pool_test.rb
|
2
|
-
|
3
|
-
require File.expand_path('../test_helper', __FILE__)
|
4
|
-
|
5
|
-
describe Beaneater::Pool do
|
6
|
-
|
7
|
-
before do
|
8
|
-
@hosts = ['localhost', 'localhost']
|
9
|
-
@bp = Beaneater::Pool.new(@hosts)
|
10
|
-
end
|
11
|
-
|
12
|
-
describe 'for #new' do
|
13
|
-
|
14
|
-
describe "for multiple connection" do
|
15
|
-
before do
|
16
|
-
@host_string_port = 'localhost:11301'
|
17
|
-
@host_num_port = '127.0.0.1:11302'
|
18
|
-
@host_string = 'host.local'
|
19
|
-
@host_num = '1.1.1.1:11303'
|
20
|
-
@hosts = [@host_string_port, @host_num_port, @host_string, @host_num]
|
21
|
-
|
22
|
-
TCPSocket.expects(:new).with('localhost',11301).once
|
23
|
-
TCPSocket.expects(:new).with('127.0.0.1',11302).once
|
24
|
-
TCPSocket.expects(:new).with('host.local',11300).once
|
25
|
-
TCPSocket.expects(:new).with('1.1.1.1',11303).once
|
26
|
-
|
27
|
-
@bp = Beaneater::Pool.new(@hosts)
|
28
|
-
end
|
29
|
-
|
30
|
-
it "should init 4 connections" do
|
31
|
-
assert_equal 4, @bp.connections.size
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
describe "for invalid connection in setup" do
|
36
|
-
it "should raise NotConnected" do
|
37
|
-
assert_raises(Beaneater::NotConnected) { Beaneater::Pool.new('localhost:5679') }
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
it "raises UnexpectedException when any Exception occurs inside the block" do
|
42
|
-
invalid_command = nil
|
43
|
-
assert_raises(Beaneater::UnknownCommandError) { @bp.transmit_to_rand(invalid_command) }
|
44
|
-
end
|
45
|
-
|
46
|
-
describe "for clearing watch list" do
|
47
|
-
it "should clear connections of tube watches" do
|
48
|
-
@bp.tubes.watch!('foo', 'bar')
|
49
|
-
assert_equal ['foo', 'bar'].sort, @bp.tubes.watched.map(&:name).sort
|
50
|
-
@bp2 = Beaneater::Pool.new(@hosts)
|
51
|
-
assert_equal ['default'], @bp2.tubes.watched.map(&:name)
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
describe "with ENV variable set" do
|
56
|
-
before do
|
57
|
-
ENV['BEANSTALKD_URL'] = '0.0.0.0:11300,127.0.0.1:11300'
|
58
|
-
end
|
59
|
-
|
60
|
-
it "should create 1 connection" do
|
61
|
-
bp = Beaneater::Pool.new
|
62
|
-
bc = bp.connections.first
|
63
|
-
bc2 = bp.connections.last
|
64
|
-
|
65
|
-
assert_equal 2, bp.connections.size
|
66
|
-
assert_equal '0.0.0.0', bc.host
|
67
|
-
assert_equal 11300, bc.port
|
68
|
-
|
69
|
-
assert_equal '127.0.0.1', bc2.host
|
70
|
-
assert_equal 11300, bc2.port
|
71
|
-
end
|
72
|
-
|
73
|
-
after do
|
74
|
-
ENV['BEANSTALKD_URL'] = nil
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
describe "by configuring via Beaneater.configure" do
|
79
|
-
before do
|
80
|
-
Beaneater.configure.beanstalkd_url = ['0.0.0.0:11300', '127.0.0.1:11300']
|
81
|
-
end
|
82
|
-
|
83
|
-
it "should create 1 connection" do
|
84
|
-
bp = Beaneater::Pool.new
|
85
|
-
bc = bp.connections.first
|
86
|
-
bc2 = bp.connections.last
|
87
|
-
|
88
|
-
assert_equal 2, bp.connections.size
|
89
|
-
assert_equal '0.0.0.0', bc.host
|
90
|
-
assert_equal 11300, bc.port
|
91
|
-
|
92
|
-
assert_equal '127.0.0.1', bc2.host
|
93
|
-
assert_equal 11300, bc2.port
|
94
|
-
end
|
95
|
-
|
96
|
-
after do
|
97
|
-
Beaneater.configure.beanstalkd_url = ['0.0.0.0:11300']
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end # new
|
101
|
-
|
102
|
-
describe 'for #transmit_to_all' do
|
103
|
-
it "should return yaml loaded response" do
|
104
|
-
res = @bp.transmit_to_all 'stats'
|
105
|
-
assert_equal 2, res.size
|
106
|
-
refute_nil res.first[:body]['current-connections']
|
107
|
-
end
|
108
|
-
end # transmit_to_all
|
109
|
-
|
110
|
-
describe 'for #transmit_to_rand' do
|
111
|
-
it "should return yaml loaded response" do
|
112
|
-
res = @bp.transmit_to_rand 'stats'
|
113
|
-
refute_nil res[:body]['current-connections']
|
114
|
-
assert_equal 'OK', res[:status]
|
115
|
-
end
|
116
|
-
|
117
|
-
it "should return id" do
|
118
|
-
Beaneater::Connection.any_instance.expects(:transmit).with("foo",{}).returns({:id => "254", :status => "INSERTED"})
|
119
|
-
res = @bp.transmit_to_rand 'foo'
|
120
|
-
assert_equal '254', res[:id]
|
121
|
-
assert_equal 'INSERTED', res[:status]
|
122
|
-
end
|
123
|
-
end # transmit_to_rand
|
124
|
-
|
125
|
-
describe 'for #transmit_until_res' do
|
126
|
-
before do
|
127
|
-
Beaneater::Connection.any_instance.expects(:transmit).with('foo', {}).twice.
|
128
|
-
returns({:status => "FAILED", :body => 'x'}).then.
|
129
|
-
returns({:status => "OK", :body => 'y'}).then.returns({:status => "OK", :body => 'z'})
|
130
|
-
end
|
131
|
-
|
132
|
-
it "should returns first matching status" do
|
133
|
-
assert_equal 'y', @bp.transmit_until_res('foo', :status => 'OK')[:body]
|
134
|
-
end
|
135
|
-
end # transmit_until_res
|
136
|
-
|
137
|
-
describe 'for #stats' do
|
138
|
-
it("should return stats object"){ assert_kind_of Beaneater::Stats, @bp.stats }
|
139
|
-
end # stats
|
140
|
-
|
141
|
-
describe 'for #tubes' do
|
142
|
-
it("should return Tubes object"){ assert_kind_of Beaneater::Tubes, @bp.tubes }
|
143
|
-
end # tubes
|
144
|
-
|
145
|
-
describe "for #safe_transmit" do
|
146
|
-
it "should retry 3 times for temporary failed connection" do
|
147
|
-
TCPSocket.any_instance.expects(:write).times(3)
|
148
|
-
TCPSocket.any_instance.expects(:gets).raises(Errno::ECONNRESET).then.
|
149
|
-
raises(Errno::ECONNRESET).then.returns('INSERTED 254').times(3)
|
150
|
-
res = @bp.transmit_to_rand "put 0 0 10 2\r\nxy"
|
151
|
-
assert_equal '254', res[:id]
|
152
|
-
assert_equal 'INSERTED', res[:status]
|
153
|
-
end
|
154
|
-
|
155
|
-
it 'should raise proper exception for invalid status NOT_FOUND' do
|
156
|
-
TCPSocket.any_instance.expects(:write).once
|
157
|
-
TCPSocket.any_instance.expects(:gets).returns('NOT_FOUND')
|
158
|
-
assert_raises(Beaneater::NotFoundError) { @bp.transmit_to_rand 'foo' }
|
159
|
-
end
|
160
|
-
|
161
|
-
it 'should raise proper exception for invalid status BAD_FORMAT' do
|
162
|
-
TCPSocket.any_instance.expects(:write).once
|
163
|
-
TCPSocket.any_instance.expects(:gets).returns('BAD_FORMAT')
|
164
|
-
assert_raises(Beaneater::BadFormatError) { @bp.transmit_to_rand 'foo' }
|
165
|
-
end
|
166
|
-
|
167
|
-
it 'should raise proper exception for invalid status DEADLINE_SOON' do
|
168
|
-
TCPSocket.any_instance.expects(:write).once
|
169
|
-
TCPSocket.any_instance.expects(:gets).once.returns('DEADLINE_SOON')
|
170
|
-
assert_raises(Beaneater::DeadlineSoonError) { @bp.transmit_to_rand 'expecting deadline' }
|
171
|
-
end
|
172
|
-
end # safe_transmit
|
173
|
-
|
174
|
-
describe "for #close" do
|
175
|
-
it "should support closing the pool" do
|
176
|
-
connection = @bp.connections.first
|
177
|
-
assert_equal 2, @bp.connections.size
|
178
|
-
assert_kind_of Beaneater::Connection, connection
|
179
|
-
assert_kind_of TCPSocket, connection.connection
|
180
|
-
@bp.close
|
181
|
-
assert_equal 0, @bp.connections.size
|
182
|
-
assert_nil connection.connection
|
183
|
-
end
|
184
|
-
end # close
|
185
|
-
end # Beaneater::Pool
|