beaneater 0.3.0 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.travis.yml +15 -0
- data/CHANGELOG.md +36 -1
- data/Gemfile +5 -1
- data/README.md +50 -26
- data/Rakefile +3 -1
- data/beaneater.gemspec +1 -1
- data/examples/demo.rb +1 -2
- data/lib/beaneater/configuration.rb +3 -1
- data/lib/beaneater/connection.rb +135 -30
- data/lib/beaneater/errors.rb +4 -4
- data/lib/beaneater/job/collection.rb +67 -37
- data/lib/beaneater/job/record.rb +42 -30
- data/lib/beaneater/stats/fast_struct.rb +1 -1
- data/lib/beaneater/stats/stat_struct.rb +8 -2
- data/lib/beaneater/stats.rb +28 -5
- data/lib/beaneater/tube/collection.rb +67 -26
- data/lib/beaneater/tube/record.rb +45 -27
- data/lib/beaneater/version.rb +2 -2
- data/lib/beaneater.rb +57 -4
- data/test/beaneater_test.rb +5 -11
- data/test/connection_test.rb +55 -1
- data/test/errors_test.rb +9 -2
- data/test/job_test.rb +8 -12
- data/test/jobs_test.rb +15 -43
- data/test/prompt_regexp_test.rb +14 -2
- data/test/stat_struct_test.rb +12 -2
- data/test/stats_test.rb +9 -9
- data/test/test_helper.rb +34 -10
- data/test/tube_test.rb +12 -23
- data/test/tubes_test.rb +61 -56
- metadata +25 -65
- data/lib/beaneater/pool.rb +0 -166
- data/lib/beaneater/pool_command.rb +0 -79
- data/test/pool_command_test.rb +0 -90
- data/test/pool_test.rb +0 -180
@@ -1,79 +0,0 @@
|
|
1
|
-
require 'set'
|
2
|
-
|
3
|
-
module Beaneater
|
4
|
-
# Represents collection of pool related commands.
|
5
|
-
class PoolCommand
|
6
|
-
# @!attribute pool
|
7
|
-
# @return [Beaneater::Pool] returns Pool object
|
8
|
-
attr_reader :pool
|
9
|
-
|
10
|
-
# Initialize new connection
|
11
|
-
#
|
12
|
-
# @param [Beaneater::Pool] pool Pool object
|
13
|
-
def initialize(pool)
|
14
|
-
@pool = pool
|
15
|
-
end
|
16
|
-
|
17
|
-
# Delegate to Pool#transmit_to_all and if needed will merge responses from beanstalkd.
|
18
|
-
#
|
19
|
-
# @param [String] body Beanstalkd command
|
20
|
-
# @param [Hash{String => String, Boolean}] options socket connections options
|
21
|
-
# @option options [Boolean] merge Ask for merging responses or not
|
22
|
-
# @param [Proc] block Block passed in socket connection object
|
23
|
-
# @example
|
24
|
-
# @pool.transmit_to_all("stats")
|
25
|
-
#
|
26
|
-
def transmit_to_all(body, options={}, &block)
|
27
|
-
merge = options.delete(:merge)
|
28
|
-
res = pool.transmit_to_all(body, options, &block)
|
29
|
-
first = res.find { |r| r && r[:status] }
|
30
|
-
if first && merge
|
31
|
-
res = { :status => first[:status], :body => sum_items(res.map { |r| r[:body] }) }
|
32
|
-
end
|
33
|
-
res
|
34
|
-
end
|
35
|
-
|
36
|
-
# Delegate missing methods to pool
|
37
|
-
# @api public
|
38
|
-
def method_missing(name, *args, &block)
|
39
|
-
if pool.respond_to?(name)
|
40
|
-
pool.send(name, *args, &block)
|
41
|
-
else # not a known pool command
|
42
|
-
super
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
protected
|
47
|
-
|
48
|
-
# Selects items from collection and then merges the individual values
|
49
|
-
# Supports array of hashes or array of arrays
|
50
|
-
#
|
51
|
-
# @param [Array<Hash, Array>] hs Collection of responses returned from beanstalkd
|
52
|
-
# @return [Hash{Symbol => String}] Merged responses combining values from all the hash bodies
|
53
|
-
# @example
|
54
|
-
# self.sum_items([{ :foo => 1, :bar => 5 }, { :foo => 2, :bar => 3 }])
|
55
|
-
# => { :foo => 3, :bar => 8 }
|
56
|
-
# self.sum_items([['foo', 'bar'], ['foo', 'bar', 'baz']])
|
57
|
-
# => ['foo', 'bar', 'baz']
|
58
|
-
#
|
59
|
-
def sum_items(items)
|
60
|
-
if items.first.is_a?(Hash)
|
61
|
-
items.select { |h| h.is_a?(Hash) }.
|
62
|
-
inject({}) { |a,b| a.merge(b) { |k,o,n| combine_stats(k, o, n) } }
|
63
|
-
elsif items.first.is_a?(Array)
|
64
|
-
items.flatten.uniq
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
# Combine two values for given key
|
69
|
-
#
|
70
|
-
# @param [String] k key name within response hash
|
71
|
-
# @return [Set,Integer] combined value for stat
|
72
|
-
# @example
|
73
|
-
# self.combine_stats('total_connections', 4, 5) # => 9
|
74
|
-
#
|
75
|
-
def combine_stats(k, a, b)
|
76
|
-
['name', 'version', 'pid'].include?(k) ? Set[a] + Set[b] : a + b
|
77
|
-
end
|
78
|
-
end # PoolCommand
|
79
|
-
end # Beaneater
|
data/test/pool_command_test.rb
DELETED
@@ -1,90 +0,0 @@
|
|
1
|
-
# test/connection_test.rb
|
2
|
-
|
3
|
-
require File.expand_path('../test_helper', __FILE__)
|
4
|
-
|
5
|
-
describe Beaneater::PoolCommand do
|
6
|
-
|
7
|
-
describe 'for #new' do
|
8
|
-
before do
|
9
|
-
@pool = stub
|
10
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
11
|
-
end
|
12
|
-
|
13
|
-
it "should store pool" do
|
14
|
-
assert_equal @pool, @command.pool
|
15
|
-
end
|
16
|
-
end #new
|
17
|
-
|
18
|
-
describe 'for #transmit_to_all' do
|
19
|
-
describe 'for regular command' do
|
20
|
-
before do
|
21
|
-
@pool = stub(:transmit_to_all => [{ :body => "foo", :status => "OK" }])
|
22
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
23
|
-
end
|
24
|
-
|
25
|
-
it "can run regular command" do
|
26
|
-
res = @command.transmit_to_all("foo")
|
27
|
-
assert_equal "OK", res[0][:status]
|
28
|
-
assert_equal "foo", res[0][:body]
|
29
|
-
end
|
30
|
-
end # regular command
|
31
|
-
|
32
|
-
describe 'for merge command with hashes' do
|
33
|
-
before do
|
34
|
-
@pool = stub(:transmit_to_all => [
|
35
|
-
{ :body => { 'x' => 1, 'version' => 1.1 }, :status => "OK"},
|
36
|
-
{ :body => { 'x' => 3,'version' => 1.2 }, :status => "OK" }
|
37
|
-
])
|
38
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
39
|
-
end
|
40
|
-
|
41
|
-
it "can run merge command " do
|
42
|
-
cmd = @command.transmit_to_all("bar", :merge => true)
|
43
|
-
assert_equal "OK", cmd[:status]
|
44
|
-
assert_equal 4, cmd[:body]['x']
|
45
|
-
assert_equal Set[1.1, 1.2], cmd[:body]['version']
|
46
|
-
end
|
47
|
-
end # merged command
|
48
|
-
|
49
|
-
describe 'for merge command with arrays' do
|
50
|
-
before do
|
51
|
-
@pool = stub(:transmit_to_all => [
|
52
|
-
{ :body => ['foo', 'bar'], :status => "OK"},
|
53
|
-
{ :body => ['foo', 'bar', 'baz'], :status => "OK" }
|
54
|
-
])
|
55
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
56
|
-
end
|
57
|
-
|
58
|
-
it "can run merge command " do
|
59
|
-
cmd = @command.transmit_to_all("bar", :merge => true)
|
60
|
-
assert_equal "OK", cmd[:status]
|
61
|
-
assert_equal ['foo', 'bar', 'baz'].sort, cmd[:body].sort
|
62
|
-
end
|
63
|
-
end # merged command
|
64
|
-
end # transmit_to_all
|
65
|
-
|
66
|
-
describe 'for #method_missing' do
|
67
|
-
describe '#transmit_to_rand' do
|
68
|
-
before do
|
69
|
-
@pool = stub
|
70
|
-
@pool.expects(:transmit_to_rand).with('foo').returns('OK')
|
71
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
72
|
-
end
|
73
|
-
|
74
|
-
it 'delegates to connection' do
|
75
|
-
assert_equal 'OK', @command.transmit_to_rand('foo')
|
76
|
-
end
|
77
|
-
end # transmit_to_rand
|
78
|
-
|
79
|
-
describe 'invalid method' do
|
80
|
-
before do
|
81
|
-
@pool = stub
|
82
|
-
@command = Beaneater::PoolCommand.new(@pool)
|
83
|
-
end
|
84
|
-
|
85
|
-
it 'raises no method error' do
|
86
|
-
assert_raises(NoMethodError) { @command.foo('foo') }
|
87
|
-
end
|
88
|
-
end # transmit_to_rand
|
89
|
-
end
|
90
|
-
end # Beaneater::PoolCommand
|
data/test/pool_test.rb
DELETED
@@ -1,180 +0,0 @@
|
|
1
|
-
# test/connection_test.rb
|
2
|
-
|
3
|
-
require File.expand_path('../test_helper', __FILE__)
|
4
|
-
|
5
|
-
describe Beaneater::Pool do
|
6
|
-
|
7
|
-
before do
|
8
|
-
@hosts = ['localhost', 'localhost']
|
9
|
-
@bp = Beaneater::Pool.new(@hosts)
|
10
|
-
end
|
11
|
-
|
12
|
-
describe 'for #new' do
|
13
|
-
|
14
|
-
describe "for multiple connection" do
|
15
|
-
before do
|
16
|
-
@host_string_port = 'localhost:11301'
|
17
|
-
@host_num_port = '127.0.0.1:11302'
|
18
|
-
@host_string = 'host.local'
|
19
|
-
@host_num = '1.1.1.1:11303'
|
20
|
-
@hosts = [@host_string_port, @host_num_port, @host_string, @host_num]
|
21
|
-
|
22
|
-
TCPSocket.expects(:new).with('localhost',11301).once
|
23
|
-
TCPSocket.expects(:new).with('127.0.0.1',11302).once
|
24
|
-
TCPSocket.expects(:new).with('host.local',11300).once
|
25
|
-
TCPSocket.expects(:new).with('1.1.1.1',11303).once
|
26
|
-
|
27
|
-
@bp = Beaneater::Pool.new(@hosts)
|
28
|
-
end
|
29
|
-
|
30
|
-
it "should init 4 connections" do
|
31
|
-
assert_equal 4, @bp.connections.size
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
describe "for invalid connection in setup" do
|
36
|
-
it "should raise NotConnected" do
|
37
|
-
assert_raises(Beaneater::NotConnected) { Beaneater::Pool.new('localhost:5679') }
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
describe "for clearing watch list" do
|
42
|
-
it "should clear connections of tube watches" do
|
43
|
-
@bp.tubes.watch!('foo', 'bar')
|
44
|
-
assert_equal ['foo', 'bar'].sort, @bp.tubes.watched.map(&:name).sort
|
45
|
-
@bp2 = Beaneater::Pool.new(@hosts)
|
46
|
-
assert_equal ['default'], @bp2.tubes.watched.map(&:name)
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
describe "with ENV variable set" do
|
51
|
-
before do
|
52
|
-
ENV['BEANSTALKD_URL'] = '0.0.0.0:11300,127.0.0.1:11300'
|
53
|
-
end
|
54
|
-
|
55
|
-
it "should create 1 connection" do
|
56
|
-
bp = Beaneater::Pool.new
|
57
|
-
bc = bp.connections.first
|
58
|
-
bc2 = bp.connections.last
|
59
|
-
|
60
|
-
assert_equal 2, bp.connections.size
|
61
|
-
assert_equal '0.0.0.0', bc.host
|
62
|
-
assert_equal 11300, bc.port
|
63
|
-
|
64
|
-
assert_equal '127.0.0.1', bc2.host
|
65
|
-
assert_equal 11300, bc2.port
|
66
|
-
end
|
67
|
-
|
68
|
-
after do
|
69
|
-
ENV['BEANSTALKD_URL'] = nil
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
describe "by configuring via Beaneater.configure" do
|
74
|
-
before do
|
75
|
-
Beaneater.configure.beanstalkd_url = ['0.0.0.0:11300', '127.0.0.1:11300']
|
76
|
-
end
|
77
|
-
|
78
|
-
it "should create 1 connection" do
|
79
|
-
bp = Beaneater::Pool.new
|
80
|
-
bc = bp.connections.first
|
81
|
-
bc2 = bp.connections.last
|
82
|
-
|
83
|
-
assert_equal 2, bp.connections.size
|
84
|
-
assert_equal '0.0.0.0', bc.host
|
85
|
-
assert_equal 11300, bc.port
|
86
|
-
|
87
|
-
assert_equal '127.0.0.1', bc2.host
|
88
|
-
assert_equal 11300, bc2.port
|
89
|
-
end
|
90
|
-
|
91
|
-
after do
|
92
|
-
Beaneater.configure.beanstalkd_url = ['0.0.0.0:11300']
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end # new
|
96
|
-
|
97
|
-
describe 'for #transmit_to_all' do
|
98
|
-
it "should return yaml loaded response" do
|
99
|
-
res = @bp.transmit_to_all 'stats'
|
100
|
-
assert_equal 2, res.size
|
101
|
-
refute_nil res.first[:body]['current-connections']
|
102
|
-
end
|
103
|
-
end # transmit_to_all
|
104
|
-
|
105
|
-
describe 'for #transmit_to_rand' do
|
106
|
-
it "should return yaml loaded response" do
|
107
|
-
res = @bp.transmit_to_rand 'stats'
|
108
|
-
refute_nil res[:body]['current-connections']
|
109
|
-
assert_equal 'OK', res[:status]
|
110
|
-
end
|
111
|
-
|
112
|
-
it "should return id" do
|
113
|
-
Beaneater::Connection.any_instance.expects(:transmit).with("foo",{}).returns({:id => "254", :status => "INSERTED"})
|
114
|
-
res = @bp.transmit_to_rand 'foo'
|
115
|
-
assert_equal '254', res[:id]
|
116
|
-
assert_equal 'INSERTED', res[:status]
|
117
|
-
end
|
118
|
-
end # transmit_to_rand
|
119
|
-
|
120
|
-
describe 'for #transmit_until_res' do
|
121
|
-
before do
|
122
|
-
Beaneater::Connection.any_instance.expects(:transmit).with('foo', {}).twice.
|
123
|
-
returns({:status => "FAILED", :body => 'x'}).then.
|
124
|
-
returns({:status => "OK", :body => 'y'}).then.returns({:status => "OK", :body => 'z'})
|
125
|
-
end
|
126
|
-
|
127
|
-
it "should returns first matching status" do
|
128
|
-
assert_equal 'y', @bp.transmit_until_res('foo', :status => 'OK')[:body]
|
129
|
-
end
|
130
|
-
end # transmit_until_res
|
131
|
-
|
132
|
-
describe 'for #stats' do
|
133
|
-
it("should return stats object"){ assert_kind_of Beaneater::Stats, @bp.stats }
|
134
|
-
end # stats
|
135
|
-
|
136
|
-
describe 'for #tubes' do
|
137
|
-
it("should return Tubes object"){ assert_kind_of Beaneater::Tubes, @bp.tubes }
|
138
|
-
end # tubes
|
139
|
-
|
140
|
-
describe "for #safe_transmit" do
|
141
|
-
it "should retry 3 times for temporary failed connection" do
|
142
|
-
TCPSocket.any_instance.expects(:write).times(3)
|
143
|
-
TCPSocket.any_instance.expects(:gets).raises(Errno::ECONNRESET).then.
|
144
|
-
raises(Errno::ECONNRESET).then.returns('INSERTED 254').times(3)
|
145
|
-
res = @bp.transmit_to_rand "put 0 0 10 2\r\nxy"
|
146
|
-
assert_equal '254', res[:id]
|
147
|
-
assert_equal 'INSERTED', res[:status]
|
148
|
-
end
|
149
|
-
|
150
|
-
it 'should raise proper exception for invalid status NOT_FOUND' do
|
151
|
-
TCPSocket.any_instance.expects(:write).once
|
152
|
-
TCPSocket.any_instance.expects(:gets).returns('NOT_FOUND')
|
153
|
-
assert_raises(Beaneater::NotFoundError) { @bp.transmit_to_rand 'foo' }
|
154
|
-
end
|
155
|
-
|
156
|
-
it 'should raise proper exception for invalid status BAD_FORMAT' do
|
157
|
-
TCPSocket.any_instance.expects(:write).once
|
158
|
-
TCPSocket.any_instance.expects(:gets).returns('BAD_FORMAT')
|
159
|
-
assert_raises(Beaneater::BadFormatError) { @bp.transmit_to_rand 'foo' }
|
160
|
-
end
|
161
|
-
|
162
|
-
it 'should raise proper exception for invalid status DEADLINE_SOON' do
|
163
|
-
TCPSocket.any_instance.expects(:write).once
|
164
|
-
TCPSocket.any_instance.expects(:gets).once.returns('DEADLINE_SOON')
|
165
|
-
assert_raises(Beaneater::DeadlineSoonError) { @bp.transmit_to_rand 'expecting deadline' }
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
describe "for #close" do
|
170
|
-
it "should support closing the pool" do
|
171
|
-
connection = @bp.connections.first
|
172
|
-
assert_equal 2, @bp.connections.size
|
173
|
-
assert_kind_of Beaneater::Connection, connection
|
174
|
-
assert_kind_of TCPSocket, connection.connection
|
175
|
-
@bp.close
|
176
|
-
assert_equal 0, @bp.connections.size
|
177
|
-
assert_nil connection.connection
|
178
|
-
end
|
179
|
-
end # close
|
180
|
-
end # Beaneater::Pool
|