beaneater 0.3.1 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.travis.yml +15 -0
- data/CHANGELOG.md +30 -1
- data/Gemfile +5 -1
- data/README.md +45 -24
- data/Rakefile +3 -1
- data/beaneater.gemspec +1 -0
- data/examples/demo.rb +1 -2
- data/lib/beaneater.rb +57 -4
- data/lib/beaneater/configuration.rb +3 -1
- data/lib/beaneater/connection.rb +128 -29
- data/lib/beaneater/errors.rb +2 -2
- data/lib/beaneater/job/collection.rb +67 -37
- data/lib/beaneater/job/record.rb +42 -30
- data/lib/beaneater/stats.rb +28 -5
- data/lib/beaneater/stats/fast_struct.rb +1 -1
- data/lib/beaneater/stats/stat_struct.rb +8 -2
- data/lib/beaneater/tube/collection.rb +67 -26
- data/lib/beaneater/tube/record.rb +45 -27
- data/lib/beaneater/version.rb +2 -2
- data/test/beaneater_test.rb +5 -11
- data/test/connection_test.rb +55 -1
- data/test/errors_test.rb +1 -1
- data/test/job_test.rb +8 -12
- data/test/jobs_test.rb +15 -43
- data/test/prompt_regexp_test.rb +14 -2
- data/test/stat_struct_test.rb +12 -2
- data/test/stats_test.rb +9 -9
- data/test/test_helper.rb +33 -7
- data/test/tube_test.rb +12 -23
- data/test/tubes_test.rb +61 -56
- metadata +23 -29
- data/lib/beaneater/pool.rb +0 -166
- data/lib/beaneater/pool_command.rb +0 -79
- data/test/pool_command_test.rb +0 -90
- data/test/pool_test.rb +0 -185
@@ -1,16 +1,36 @@
|
|
1
|
-
|
1
|
+
class Beaneater
|
2
2
|
# Represents collection of tube related commands.
|
3
|
-
|
3
|
+
|
4
|
+
class Tubes
|
5
|
+
include Enumerable
|
6
|
+
|
7
|
+
# @!attribute client
|
8
|
+
# @return [Beaneater] returns the client instance
|
9
|
+
attr_reader :client
|
4
10
|
|
5
11
|
# Creates new tubes instance.
|
6
12
|
#
|
7
|
-
# @param [Beaneater
|
13
|
+
# @param [Beaneater] client The beaneater client instance.
|
8
14
|
# @example
|
9
|
-
# Beaneater::Tubes.new(@
|
15
|
+
# Beaneater::Tubes.new(@client)
|
16
|
+
#
|
17
|
+
def initialize(client)
|
18
|
+
@client = client
|
19
|
+
end
|
20
|
+
|
21
|
+
def last_used
|
22
|
+
client.connection.tube_used
|
23
|
+
end
|
24
|
+
|
25
|
+
def last_used=(tube_name)
|
26
|
+
client.connection.tube_used = tube_name
|
27
|
+
end
|
28
|
+
|
29
|
+
# Delegates transmit to the connection object.
|
10
30
|
#
|
11
|
-
|
12
|
-
|
13
|
-
|
31
|
+
# @see Beaneater::Connection#transmit
|
32
|
+
def transmit(command, **options)
|
33
|
+
client.connection.transmit(command, **options)
|
14
34
|
end
|
15
35
|
|
16
36
|
# Finds the specified beanstalk tube.
|
@@ -24,7 +44,7 @@ module Beaneater
|
|
24
44
|
#
|
25
45
|
# @api public
|
26
46
|
def find(tube_name)
|
27
|
-
Tube.new(
|
47
|
+
Tube.new(client, tube_name)
|
28
48
|
end
|
29
49
|
alias_method :[], :find
|
30
50
|
|
@@ -35,13 +55,14 @@ module Beaneater
|
|
35
55
|
# @yield [job] Reserved beaneater job.
|
36
56
|
# @return [Beaneater::Job] Reserved beaneater job.
|
37
57
|
# @example
|
38
|
-
# @
|
58
|
+
# @client.tubes.reserve { |job| process(job) }
|
39
59
|
# # => <Beaneater::Job id=5 body="foo">
|
40
60
|
#
|
41
61
|
# @api public
|
42
62
|
def reserve(timeout=nil, &block)
|
43
|
-
res =
|
44
|
-
|
63
|
+
res = transmit(
|
64
|
+
timeout ? "reserve-with-timeout #{timeout}" : 'reserve')
|
65
|
+
job = Job.new(client, res)
|
45
66
|
block.call(job) if block_given?
|
46
67
|
job
|
47
68
|
end
|
@@ -50,36 +71,54 @@ module Beaneater
|
|
50
71
|
#
|
51
72
|
# @return [Array<Beaneater::Tube>] List of all beanstalk tubes.
|
52
73
|
# @example
|
53
|
-
# @
|
74
|
+
# @client.tubes.all
|
54
75
|
# # => [<Beaneater::Tube name="tube2">, <Beaneater::Tube name="tube3">]
|
55
76
|
#
|
56
77
|
# @api public
|
57
78
|
def all
|
58
|
-
|
79
|
+
transmit('list-tubes')[:body].map do |tube_name|
|
80
|
+
Tube.new(client, tube_name)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
# Calls the given block once for each known beanstalk tube, passing that element as a parameter.
|
85
|
+
#
|
86
|
+
# @return An Enumerator is returned if no block is given.
|
87
|
+
# @example
|
88
|
+
# @pool.tubes.each {|t| puts t.name}
|
89
|
+
#
|
90
|
+
# @api public
|
91
|
+
def each(&block)
|
92
|
+
all.each(&block)
|
59
93
|
end
|
60
94
|
|
61
95
|
# List of watched beanstalk tubes.
|
62
96
|
#
|
63
97
|
# @return [Array<Beaneater::Tube>] List of watched beanstalk tubes.
|
64
98
|
# @example
|
65
|
-
# @
|
99
|
+
# @client.tubes.watched
|
66
100
|
# # => [<Beaneater::Tube name="tube2">, <Beaneater::Tube name="tube3">]
|
67
101
|
#
|
68
102
|
# @api public
|
69
103
|
def watched
|
70
|
-
|
104
|
+
last_watched = transmit('list-tubes-watched')[:body]
|
105
|
+
client.connection.tubes_watched = last_watched.dup
|
106
|
+
last_watched.map do |tube_name|
|
107
|
+
Tube.new(client, tube_name)
|
108
|
+
end
|
71
109
|
end
|
72
110
|
|
73
111
|
# Currently used beanstalk tube.
|
74
112
|
#
|
75
113
|
# @return [Beaneater::Tube] Currently used beanstalk tube.
|
76
114
|
# @example
|
77
|
-
# @
|
115
|
+
# @client.tubes.used
|
78
116
|
# # => <Beaneater::Tube name="tube2">
|
79
117
|
#
|
80
118
|
# @api public
|
81
119
|
def used
|
82
|
-
|
120
|
+
last_used = transmit('list-tube-used')[:id]
|
121
|
+
Tube.new(client, last_used)
|
83
122
|
end
|
84
123
|
|
85
124
|
# Add specified beanstalkd tubes as watched.
|
@@ -87,12 +126,13 @@ module Beaneater
|
|
87
126
|
# @param [*String] names Name of tubes to watch
|
88
127
|
# @raise [Beaneater::InvalidTubeName] Tube to watch was invalid.
|
89
128
|
# @example
|
90
|
-
# @
|
129
|
+
# @client.tubes.watch('foo', 'bar')
|
91
130
|
#
|
92
131
|
# @api public
|
93
132
|
def watch(*names)
|
94
133
|
names.each do |t|
|
95
|
-
|
134
|
+
transmit "watch #{t}"
|
135
|
+
client.connection.add_to_watched(t)
|
96
136
|
end
|
97
137
|
rescue BadFormatError => ex
|
98
138
|
raise InvalidTubeName, "Tube in '#{ex.cmd}' is invalid!"
|
@@ -103,7 +143,7 @@ module Beaneater
|
|
103
143
|
# @param [*String] names Name of tubes to watch
|
104
144
|
# @raise [Beaneater::InvalidTubeName] Tube to watch was invalid.
|
105
145
|
# @example
|
106
|
-
# @
|
146
|
+
# @client.tubes.watch!('foo', 'bar')
|
107
147
|
#
|
108
148
|
# @api public
|
109
149
|
def watch!(*names)
|
@@ -116,12 +156,13 @@ module Beaneater
|
|
116
156
|
#
|
117
157
|
# @param [*String] names Name of tubes to ignore
|
118
158
|
# @example
|
119
|
-
# @
|
159
|
+
# @client.tubes.ignore('foo', 'bar')
|
120
160
|
#
|
121
161
|
# @api public
|
122
162
|
def ignore(*names)
|
123
163
|
names.each do |w|
|
124
|
-
|
164
|
+
transmit "ignore #{w}"
|
165
|
+
client.connection.remove_from_watched(w)
|
125
166
|
end
|
126
167
|
end
|
127
168
|
|
@@ -132,11 +173,11 @@ module Beaneater
|
|
132
173
|
# @conn.tubes.use("some-tube")
|
133
174
|
#
|
134
175
|
def use(tube)
|
135
|
-
return tube if
|
136
|
-
|
137
|
-
|
176
|
+
return tube if last_used == tube
|
177
|
+
transmit("use #{tube}")
|
178
|
+
self.last_used = tube
|
138
179
|
rescue BadFormatError
|
139
180
|
raise InvalidTubeName, "Tube cannot be named '#{tube}'"
|
140
181
|
end
|
141
182
|
end # Tubes
|
142
|
-
end # Beaneater
|
183
|
+
end # Beaneater
|
@@ -1,22 +1,31 @@
|
|
1
|
-
|
1
|
+
class Beaneater
|
2
2
|
# Beanstalk tube which contains jobs which can be inserted, reserved, et al.
|
3
|
-
class Tube
|
3
|
+
class Tube
|
4
4
|
|
5
5
|
# @!attribute name
|
6
6
|
# @return [String] name of the tube
|
7
|
-
|
7
|
+
# @!attribute client
|
8
|
+
# @return [Beaneater] returns the client instance
|
9
|
+
attr_reader :name, :client
|
8
10
|
|
9
11
|
# Fetches the specified tube.
|
10
12
|
#
|
11
|
-
# @param [Beaneater
|
13
|
+
# @param [Beaneater] client The beaneater client instance.
|
12
14
|
# @param [String] name The name for this tube.
|
13
15
|
# @example
|
14
|
-
# Beaneater::Tube.new(@
|
16
|
+
# Beaneater::Tube.new(@client, 'tube-name')
|
15
17
|
#
|
16
|
-
def initialize(
|
18
|
+
def initialize(client, name)
|
19
|
+
@client = client
|
17
20
|
@name = name.to_s
|
18
21
|
@mutex = Mutex.new
|
19
|
-
|
22
|
+
end
|
23
|
+
|
24
|
+
# Delegates transmit to the connection object.
|
25
|
+
#
|
26
|
+
# @see Beaneater::Connection#transmit
|
27
|
+
def transmit(command, options={})
|
28
|
+
client.connection.transmit(command, **options)
|
20
29
|
end
|
21
30
|
|
22
31
|
# Inserts job with specified body onto tube.
|
@@ -33,10 +42,16 @@ module Beaneater
|
|
33
42
|
# @api public
|
34
43
|
def put(body, options={})
|
35
44
|
safe_use do
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
45
|
+
serialized_body = config.job_serializer.call(body)
|
46
|
+
|
47
|
+
options = {
|
48
|
+
:pri => config.default_put_pri,
|
49
|
+
:delay => config.default_put_delay,
|
50
|
+
:ttr => config.default_put_ttr
|
51
|
+
}.merge(options)
|
52
|
+
|
53
|
+
cmd_options = "#{options[:pri]} #{options[:delay]} #{options[:ttr]} #{serialized_body.bytesize}"
|
54
|
+
transmit("put #{cmd_options}\r\n#{serialized_body}")
|
40
55
|
end
|
41
56
|
end
|
42
57
|
|
@@ -50,10 +65,10 @@ module Beaneater
|
|
50
65
|
# @api public
|
51
66
|
def peek(state)
|
52
67
|
safe_use do
|
53
|
-
res =
|
54
|
-
Job.new(res)
|
68
|
+
res = transmit("peek-#{state}")
|
69
|
+
Job.new(client, res)
|
55
70
|
end
|
56
|
-
rescue Beaneater::NotFoundError
|
71
|
+
rescue Beaneater::NotFoundError
|
57
72
|
# Return nil if not found
|
58
73
|
nil
|
59
74
|
end
|
@@ -69,8 +84,8 @@ module Beaneater
|
|
69
84
|
#
|
70
85
|
# @api public
|
71
86
|
def reserve(timeout=nil, &block)
|
72
|
-
|
73
|
-
|
87
|
+
client.tubes.watch!(self.name)
|
88
|
+
client.tubes.reserve(timeout, &block)
|
74
89
|
end
|
75
90
|
|
76
91
|
# Kick specified number of jobs from buried to ready state.
|
@@ -82,18 +97,18 @@ module Beaneater
|
|
82
97
|
#
|
83
98
|
# @api public
|
84
99
|
def kick(bounds=1)
|
85
|
-
safe_use {
|
100
|
+
safe_use { transmit("kick #{bounds}") }
|
86
101
|
end
|
87
102
|
|
88
103
|
# Returns related stats for this tube.
|
89
104
|
#
|
90
105
|
# @return [Beaneater::StatStruct] Struct of tube related values
|
91
106
|
# @example
|
92
|
-
# @tube.stats.
|
107
|
+
# @tube.stats.current_jobs_delayed # => 24
|
93
108
|
#
|
94
109
|
# @api public
|
95
110
|
def stats
|
96
|
-
res =
|
111
|
+
res = transmit("stats-tube #{name}")
|
97
112
|
StatStruct.from_hash(res[:body])
|
98
113
|
end
|
99
114
|
|
@@ -106,7 +121,7 @@ module Beaneater
|
|
106
121
|
#
|
107
122
|
# @api public
|
108
123
|
def pause(delay)
|
109
|
-
|
124
|
+
transmit("pause-tube #{name} #{delay}")
|
110
125
|
end
|
111
126
|
|
112
127
|
# Clears all unreserved jobs in all states from the tube
|
@@ -115,14 +130,18 @@ module Beaneater
|
|
115
130
|
# @tube.clear
|
116
131
|
#
|
117
132
|
def clear
|
118
|
-
|
133
|
+
client.tubes.watch!(self.name)
|
119
134
|
%w(delayed buried ready).each do |state|
|
120
135
|
while job = self.peek(state.to_sym)
|
121
|
-
|
136
|
+
begin
|
137
|
+
job.delete
|
138
|
+
rescue Beaneater::UnexpectedResponse, Beaneater::NotFoundError
|
139
|
+
# swallow any issues
|
140
|
+
end
|
122
141
|
end
|
123
142
|
end
|
124
|
-
|
125
|
-
rescue Beaneater::
|
143
|
+
client.tubes.ignore(name)
|
144
|
+
rescue Beaneater::NotIgnoredError
|
126
145
|
# swallow any issues
|
127
146
|
end
|
128
147
|
|
@@ -144,12 +163,12 @@ module Beaneater
|
|
144
163
|
# @param [Proc] block Beanstalk command to transmit.
|
145
164
|
# @return [Object] Result of block passed
|
146
165
|
# @example
|
147
|
-
# safe_use {
|
166
|
+
# safe_use { transmit("kick 1") }
|
148
167
|
# # => "Response to kick command"
|
149
168
|
#
|
150
169
|
def safe_use(&block)
|
151
170
|
@mutex.lock
|
152
|
-
tubes.use(self.name)
|
171
|
+
client.tubes.use(self.name)
|
153
172
|
yield
|
154
173
|
ensure
|
155
174
|
@mutex.unlock
|
@@ -161,6 +180,5 @@ module Beaneater
|
|
161
180
|
def config
|
162
181
|
Beaneater.configuration
|
163
182
|
end
|
164
|
-
|
165
183
|
end # Tube
|
166
184
|
end # Beaneater
|
data/lib/beaneater/version.rb
CHANGED
data/test/beaneater_test.rb
CHANGED
@@ -2,8 +2,7 @@ require File.expand_path('../test_helper', __FILE__)
|
|
2
2
|
|
3
3
|
describe "beanstalk-client" do
|
4
4
|
before do
|
5
|
-
@beanstalk = Beaneater
|
6
|
-
# @beanstalk = Beaneater::Pool.new(['127.0.0.1:11300', '127.0.0.1:11301'])
|
5
|
+
@beanstalk = Beaneater.new('127.0.0.1:11300')
|
7
6
|
@tubes = ['one', 'two', 'three']
|
8
7
|
|
9
8
|
# Put something on each tube so they exist
|
@@ -23,12 +22,12 @@ describe "beanstalk-client" do
|
|
23
22
|
# A: put one
|
24
23
|
a = Thread.new do
|
25
24
|
tube_one = @beanstalk.tubes.find('one')
|
26
|
-
sleep
|
25
|
+
sleep 0.5
|
27
26
|
tube_one.put('one')
|
28
27
|
end
|
29
28
|
|
30
29
|
b = Thread.new do
|
31
|
-
sleep
|
30
|
+
sleep 0.125
|
32
31
|
tube_two = @beanstalk.tubes.find('two')
|
33
32
|
tube_two.put('two')
|
34
33
|
end
|
@@ -52,13 +51,13 @@ describe "beanstalk-client" do
|
|
52
51
|
before do
|
53
52
|
a = Thread.new do
|
54
53
|
tube_one = @beanstalk.tubes.find('one')
|
55
|
-
sleep
|
54
|
+
sleep 0.5
|
56
55
|
tube_one.put('one')
|
57
56
|
end
|
58
57
|
|
59
58
|
b = Thread.new do
|
60
59
|
tube_two = @beanstalk.tubes.find('two')
|
61
|
-
sleep
|
60
|
+
sleep 0.125
|
62
61
|
tube_two.put('two')
|
63
62
|
end
|
64
63
|
|
@@ -108,9 +107,4 @@ describe "beanstalk-client" do
|
|
108
107
|
assert_nil @beanstalk.jobs.find(@job.id)
|
109
108
|
end
|
110
109
|
end
|
111
|
-
|
112
|
-
after do
|
113
|
-
cleanup_tubes!(@tubes, @beanstalk)
|
114
|
-
end
|
115
|
-
|
116
110
|
end
|
data/test/connection_test.rb
CHANGED
@@ -18,13 +18,25 @@ describe Beaneater::Connection do
|
|
18
18
|
|
19
19
|
it "should init connection" do
|
20
20
|
assert_kind_of TCPSocket, @bc.connection
|
21
|
-
|
21
|
+
if @bc.connection.peeraddr[0] == 'AF_INET'
|
22
|
+
assert_equal '127.0.0.1', @bc.connection.peeraddr[3]
|
23
|
+
else
|
24
|
+
assert_equal 'AF_INET6', @bc.connection.peeraddr[0]
|
25
|
+
assert_equal '::1', @bc.connection.peeraddr[3]
|
26
|
+
end
|
22
27
|
assert_equal 11300, @bc.connection.peeraddr[1]
|
23
28
|
end
|
24
29
|
|
25
30
|
it "should raise on invalid connection" do
|
26
31
|
assert_raises(Beaneater::NotConnected) { Beaneater::Connection.new("localhost:8544") }
|
27
32
|
end
|
33
|
+
|
34
|
+
it "should support array connection to single connection" do
|
35
|
+
@bc2 = Beaneater::Connection.new([@host])
|
36
|
+
assert_equal 'localhost', @bc.address
|
37
|
+
assert_equal 'localhost', @bc.host
|
38
|
+
assert_equal 11300, @bc.port
|
39
|
+
end
|
28
40
|
end # new
|
29
41
|
|
30
42
|
describe 'for #transmit' do
|
@@ -60,6 +72,48 @@ describe Beaneater::Connection do
|
|
60
72
|
res = @bc.transmit "put 0 0 100 256\r\n"+(0..255).to_a.pack("c*")
|
61
73
|
assert_equal 'INSERTED', res[:status]
|
62
74
|
end
|
75
|
+
|
76
|
+
it "should retry command with success after one connection failure" do
|
77
|
+
TCPSocket.any_instance.expects(:readline).times(2).
|
78
|
+
raises(EOFError.new).then.
|
79
|
+
returns("DELETED 56\nFOO")
|
80
|
+
|
81
|
+
res = @bc.transmit "delete 56\r\n"
|
82
|
+
assert_equal 'DELETED', res[:status]
|
83
|
+
end
|
84
|
+
|
85
|
+
it "should fail after exceeding retries with DrainingError" do
|
86
|
+
TCPSocket.any_instance.expects(:readline).times(3).
|
87
|
+
raises(Beaneater::UnexpectedResponse.from_status("DRAINING", "delete 56"))
|
88
|
+
|
89
|
+
assert_raises(Beaneater::DrainingError) { @bc.transmit "delete 56\r\n" }
|
90
|
+
end
|
91
|
+
|
92
|
+
it "should fail after exceeding reconnect max retries" do
|
93
|
+
# next connection attempts should fail
|
94
|
+
TCPSocket.stubs(:new).times(3).raises(Errno::ECONNREFUSED.new)
|
95
|
+
TCPSocket.any_instance.stubs(:readline).times(1).raises(EOFError.new)
|
96
|
+
|
97
|
+
assert_raises(Beaneater::NotConnected) { @bc.transmit "delete 56\r\n" }
|
98
|
+
end
|
99
|
+
|
100
|
+
it "tubes_watched are restored after reconnect" do
|
101
|
+
client = Beaneater.new('127.0.0.1:11300')
|
102
|
+
client.tubes.watch! "another"
|
103
|
+
|
104
|
+
TCPSocket.prepend Module.new {
|
105
|
+
def readline
|
106
|
+
if !$called
|
107
|
+
$called = true
|
108
|
+
raise EOFError
|
109
|
+
end
|
110
|
+
|
111
|
+
super
|
112
|
+
end
|
113
|
+
}
|
114
|
+
|
115
|
+
assert_equal %w[another], client.tubes.watched.map(&:name)
|
116
|
+
end
|
63
117
|
end # transmit
|
64
118
|
|
65
119
|
describe 'for #close' do
|