em-jack 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/COPYING +21 -0
- data/README.rdoc +80 -0
- data/lib/em-jack.rb +12 -0
- data/lib/em-jack/beanstalk_connection.rb +30 -0
- data/lib/em-jack/connection.rb +231 -0
- data/lib/em-jack/errors.rb +7 -0
- data/lib/em-jack/job.rb +23 -0
- metadata +74 -0
data/COPYING
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
Copyright (c) 2009, dan sinclair. All rights reserved.
|
2
|
+
|
3
|
+
Redistribution and use in source and binary forms, with or without
|
4
|
+
modification, are permitted provided that the following conditions are
|
5
|
+
met:
|
6
|
+
* Redistributions of source code must retain the above copyright
|
7
|
+
notice, this list of conditions and the following disclaimer.
|
8
|
+
* Redistributions in binary form must reproduce the above copyright
|
9
|
+
notice, this list of conditions and the following disclaimer in the
|
10
|
+
documentation and/or other materials provided with the distribution.
|
11
|
+
|
12
|
+
THIS SOFTWARE IS PROVIDED BY dan sinclair ''AS IS'' AND ANY
|
13
|
+
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
14
|
+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
15
|
+
DISCLAIMED. IN NO EVENT SHALL dan sinclair BE LIABLE FOR ANY
|
16
|
+
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
17
|
+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
18
|
+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
19
|
+
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
20
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
21
|
+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
= EMJack
|
2
|
+
An attempt to wrap portions of the Beanstalk protocol with EventMachine. Every command
|
3
|
+
will return a deferrable object. That object will succeed or fail depending on the
|
4
|
+
reply returned from Beanstalk.
|
5
|
+
|
6
|
+
The current, default, errback is to print the error message received.
|
7
|
+
|
8
|
+
One thing to keep in mind. The Beanstalk protocol executes all commands serially.
|
9
|
+
So, if you send a reserve command and there are no jobs Beanstalk _won't_ process
|
10
|
+
any of the commands that come after the reserve until the reserve completes.
|
11
|
+
|
12
|
+
This is a bit of a gotcha when sending a series of reserve, delete, etc and there
|
13
|
+
are no available jobs.
|
14
|
+
|
15
|
+
= Dependencies
|
16
|
+
- EventMachine
|
17
|
+
- RSpec (to run the tests)
|
18
|
+
|
19
|
+
= Examples
|
20
|
+
EM.run {
|
21
|
+
jack = EMJack::Connection.new
|
22
|
+
|
23
|
+
r = jack.use('mytube')
|
24
|
+
r.callback { |tube| puts "Using #{tube}" }
|
25
|
+
|
26
|
+
r = jack.reserve
|
27
|
+
r.callback do |job|
|
28
|
+
puts job.jobid
|
29
|
+
process(job)
|
30
|
+
|
31
|
+
r2 = jack.delete(job)
|
32
|
+
r2.callback { puts "Successfully deleted" }
|
33
|
+
end
|
34
|
+
|
35
|
+
r = jack.put("my message", :ttr => 300)
|
36
|
+
r.callback { |jobid| puts "put successful #{jobid}" }
|
37
|
+
|
38
|
+
r = jack.stats
|
39
|
+
r.callback { |stats| puts "Server up for #{stats['uptime']} seconds" "}
|
40
|
+
|
41
|
+
r = jack.stats(:tube, "mytube")
|
42
|
+
r.callback { |stats| puts "Total jobs #{stats['total-jobs']}" }
|
43
|
+
|
44
|
+
r = jack.list(:tubes)
|
45
|
+
r.callback { |tubes| puts "There are #{tubes.length} tubes defined" }
|
46
|
+
}
|
47
|
+
|
48
|
+
|
49
|
+
EMJack#each_job is useful for scenarios where the client is a job worker
|
50
|
+
and intended to process jobs continuously as they become available. Once
|
51
|
+
the queue is empty, the client will block and wait for a new job.
|
52
|
+
|
53
|
+
If multiple workers connect to the queue Beanstalkd will round-robin between
|
54
|
+
the workers.
|
55
|
+
EM.run {
|
56
|
+
jack = EMJack::Connection.new
|
57
|
+
|
58
|
+
jack.each_job do |job|
|
59
|
+
puts "Got job ##{job.jobid}: #{job}"
|
60
|
+
|
61
|
+
if process(job)
|
62
|
+
r = jack.delete(job)
|
63
|
+
r.callback { puts "*Deleted #{job}*" }
|
64
|
+
else
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def process(job)
|
69
|
+
# Some kind of job processing
|
70
|
+
end
|
71
|
+
}
|
72
|
+
|
73
|
+
|
74
|
+
= Contact
|
75
|
+
If you've got any questions, comments or bugs, please let me know. You can
|
76
|
+
contact me by email at dj2 at everburning dot com.
|
77
|
+
|
78
|
+
|
79
|
+
= Contributors
|
80
|
+
Peter Kieltyka (EMJack#each_job)
|
data/lib/em-jack.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
|
3
|
+
module EMJack
|
4
|
+
class BeanstalkConnection < EM::Connection
|
5
|
+
attr_accessor :client
|
6
|
+
|
7
|
+
def connection_completed
|
8
|
+
@client.connected
|
9
|
+
end
|
10
|
+
|
11
|
+
def receive_data(data)
|
12
|
+
@client.received(data)
|
13
|
+
end
|
14
|
+
|
15
|
+
def send(command, *args)
|
16
|
+
cmd = command.to_s
|
17
|
+
cmd << " #{args.join(" ")}" unless args.length == 0
|
18
|
+
cmd << "\r\n"
|
19
|
+
send_data(cmd)
|
20
|
+
end
|
21
|
+
|
22
|
+
def send_with_data(command, data, *args)
|
23
|
+
send_data("#{command.to_s} #{args.join(" ")}\r\n#{data}\r\n")
|
24
|
+
end
|
25
|
+
|
26
|
+
def unbind
|
27
|
+
@client.disconnected
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,231 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module EMJack
|
5
|
+
class Connection
|
6
|
+
RETRY_COUNT = 5
|
7
|
+
|
8
|
+
attr_accessor :host, :port
|
9
|
+
|
10
|
+
def initialize(opts = {})
|
11
|
+
@host = opts[:host] || 'localhost'
|
12
|
+
@port = opts[:port] || 11300
|
13
|
+
@tube = opts[:tube]
|
14
|
+
|
15
|
+
@used_tube = 'default'
|
16
|
+
@watched_tubes = ['default']
|
17
|
+
|
18
|
+
@data = ""
|
19
|
+
@retries = 0
|
20
|
+
@in_reserve = false
|
21
|
+
@deferrables = []
|
22
|
+
|
23
|
+
@conn = EM::connect(host, port, EMJack::BeanstalkConnection) do |conn|
|
24
|
+
conn.client = self
|
25
|
+
end
|
26
|
+
|
27
|
+
unless @tube.nil?
|
28
|
+
use(@tube)
|
29
|
+
watch(@tube)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def use(tube)
|
34
|
+
return if @used_tube == tube
|
35
|
+
@used_tube = tube
|
36
|
+
@conn.send(:use, tube)
|
37
|
+
add_deferrable
|
38
|
+
end
|
39
|
+
|
40
|
+
def watch(tube)
|
41
|
+
return if @watched_tubes.include?(tube)
|
42
|
+
@watched_tubes.push(tube)
|
43
|
+
@conn.send(:watch, tube)
|
44
|
+
add_deferrable
|
45
|
+
end
|
46
|
+
|
47
|
+
def ignore(tube)
|
48
|
+
return if not @watched_tubes.include?(tube)
|
49
|
+
@watched_tubes.delete(tube)
|
50
|
+
@conn.send(:ignore, tube)
|
51
|
+
add_deferrable
|
52
|
+
end
|
53
|
+
|
54
|
+
def reserve(timeout = nil)
|
55
|
+
if timeout
|
56
|
+
@conn.send(:'reserve-with-timeout', timeout)
|
57
|
+
else
|
58
|
+
@conn.send(:reserve)
|
59
|
+
end
|
60
|
+
add_deferrable
|
61
|
+
end
|
62
|
+
|
63
|
+
def each_job(&block)
|
64
|
+
work = Proc.new do
|
65
|
+
r = reserve
|
66
|
+
r.callback do |job|
|
67
|
+
block.call(job)
|
68
|
+
EM.next_tick { work.call }
|
69
|
+
end
|
70
|
+
end
|
71
|
+
work.call
|
72
|
+
end
|
73
|
+
|
74
|
+
def stats(type = nil, val = nil)
|
75
|
+
case(type)
|
76
|
+
when nil then @conn.send(:stats)
|
77
|
+
when :tube then @conn.send(:'stats-tube', val)
|
78
|
+
when :job then @conn.send(:'stats-job', val.jobid)
|
79
|
+
else raise EMJack::InvalidCommand.new
|
80
|
+
end
|
81
|
+
add_deferrable
|
82
|
+
end
|
83
|
+
|
84
|
+
def list(type = nil)
|
85
|
+
case(type)
|
86
|
+
when nil then @conn.send(:'list-tubes')
|
87
|
+
when :used then @conn.send(:'list-tube-used')
|
88
|
+
when :watched then @conn.send(:'list-tubes-watched')
|
89
|
+
else raise EMJack::InvalidCommand.new
|
90
|
+
end
|
91
|
+
add_deferrable
|
92
|
+
end
|
93
|
+
|
94
|
+
def delete(job)
|
95
|
+
return if job.nil?
|
96
|
+
@conn.send(:delete, job.jobid)
|
97
|
+
add_deferrable
|
98
|
+
end
|
99
|
+
|
100
|
+
def release(job)
|
101
|
+
return if job.nil?
|
102
|
+
@conn.send(:release, job.jobid, 0, 0)
|
103
|
+
add_deferrable
|
104
|
+
end
|
105
|
+
|
106
|
+
def put(msg, opts = {})
|
107
|
+
pri = (opts[:priority] || 65536).to_i
|
108
|
+
if pri< 0
|
109
|
+
pri = 65536
|
110
|
+
elsif pri > (2 ** 32)
|
111
|
+
pri = 2 ** 32
|
112
|
+
end
|
113
|
+
|
114
|
+
delay = (opts[:delay] || 0).to_i
|
115
|
+
delay = 0 if delay < 0
|
116
|
+
|
117
|
+
ttr = (opts[:ttr] || 300).to_i
|
118
|
+
ttr = 300 if ttr < 0
|
119
|
+
|
120
|
+
m = msg.to_s
|
121
|
+
|
122
|
+
@conn.send_with_data(:put, m, pri, delay, ttr, m.length)
|
123
|
+
add_deferrable
|
124
|
+
end
|
125
|
+
|
126
|
+
def connected
|
127
|
+
@retries = 0
|
128
|
+
end
|
129
|
+
|
130
|
+
def disconnected
|
131
|
+
# XXX I think I need to run out the deferrables as failed here
|
132
|
+
# since the connection was dropped
|
133
|
+
|
134
|
+
raise EMJack::Disconnected if @retries >= RETRY_COUNT
|
135
|
+
@retries += 1
|
136
|
+
EM.add_timer(1) { @conn.reconnect(@host, @port) }
|
137
|
+
end
|
138
|
+
|
139
|
+
def add_deferrable
|
140
|
+
df = EM::DefaultDeferrable.new
|
141
|
+
df.errback { |err| puts "ERROR: #{err}" }
|
142
|
+
|
143
|
+
@deferrables.push(df)
|
144
|
+
df
|
145
|
+
end
|
146
|
+
|
147
|
+
def received(data)
|
148
|
+
@data << data
|
149
|
+
|
150
|
+
until @data.empty?
|
151
|
+
idx = @data.index(/\r\n/)
|
152
|
+
break if idx.nil?
|
153
|
+
|
154
|
+
first = @data[0..(idx + 1)]
|
155
|
+
|
156
|
+
handled = false
|
157
|
+
%w(OUT_OF_MEMORY INTERNAL_ERROR DRAINING BAD_FORMAT
|
158
|
+
UNKNOWN_COMMAND EXPECTED_CRLF JOB_TOO_BIG DEADLINE_SOON
|
159
|
+
TIMED_OUT NOT_FOUND).each do |cmd|
|
160
|
+
next unless first =~ /^#{cmd}\r\n/i
|
161
|
+
df = @deferrables.shift
|
162
|
+
df.fail(cmd.downcase.to_sym)
|
163
|
+
|
164
|
+
@data = @data[(cmd.length + 2)..-1]
|
165
|
+
handled = true
|
166
|
+
break
|
167
|
+
end
|
168
|
+
next if handled
|
169
|
+
|
170
|
+
case (first)
|
171
|
+
when /^DELETED\r\n/ then
|
172
|
+
df = @deferrables.shift
|
173
|
+
df.succeed
|
174
|
+
|
175
|
+
when /^INSERTED\s+(\d+)\r\n/ then
|
176
|
+
df = @deferrables.shift
|
177
|
+
df.succeed($1.to_i)
|
178
|
+
|
179
|
+
when /^BURIED\s+(\d+)\r\n/ then
|
180
|
+
df = @deferrables.shift
|
181
|
+
df.fail(:buried, $1.to_i)
|
182
|
+
|
183
|
+
when /^USING\s+(.*)\r\n/ then
|
184
|
+
df = @deferrables.shift
|
185
|
+
df.succeed($1)
|
186
|
+
|
187
|
+
when /^WATCHING\s+(\d+)\r\n/ then
|
188
|
+
df = @deferrables.shift
|
189
|
+
df.succeed($1.to_i)
|
190
|
+
|
191
|
+
when /^OK\s+(\d+)\r\n/ then
|
192
|
+
bytes = $1.to_i
|
193
|
+
|
194
|
+
body, @data = extract_body(bytes, @data)
|
195
|
+
break if body.nil?
|
196
|
+
|
197
|
+
df = @deferrables.shift
|
198
|
+
df.succeed(YAML.load(body))
|
199
|
+
next
|
200
|
+
|
201
|
+
when /^RESERVED\s+(\d+)\s+(\d+)\r\n/ then
|
202
|
+
id = $1.to_i
|
203
|
+
bytes = $2.to_i
|
204
|
+
|
205
|
+
body, @data = extract_body(bytes, @data)
|
206
|
+
break if body.nil?
|
207
|
+
|
208
|
+
df = @deferrables.shift
|
209
|
+
job = EMJack::Job.new(self, id, body)
|
210
|
+
df.succeed(job)
|
211
|
+
next
|
212
|
+
|
213
|
+
else
|
214
|
+
break
|
215
|
+
end
|
216
|
+
|
217
|
+
@data = @data[(@data.index(/\r\n/) + 2)..-1]
|
218
|
+
@data = "" if @data.nil?
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def extract_body(bytes, data)
|
223
|
+
rem = data[(data.index(/\r\n/) + 2)..-1]
|
224
|
+
return [nil, data] if rem.length < bytes
|
225
|
+
body = rem[0..(bytes - 1)]
|
226
|
+
data = rem[(bytes + 2)..-1]
|
227
|
+
data = "" if data.nil?
|
228
|
+
[body, data]
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
data/lib/em-jack/job.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
module EMJack
|
2
|
+
class Job
|
3
|
+
attr_accessor :jobid, :body, :ttr, :conn
|
4
|
+
|
5
|
+
def initialize(conn, jobid, body)
|
6
|
+
@conn = conn
|
7
|
+
@jobid = jobid.to_i
|
8
|
+
@body = body
|
9
|
+
end
|
10
|
+
|
11
|
+
def delete
|
12
|
+
@conn.delete(self)
|
13
|
+
end
|
14
|
+
|
15
|
+
def stats
|
16
|
+
@conn.stats(:job, self)
|
17
|
+
end
|
18
|
+
|
19
|
+
def to_s
|
20
|
+
"#{@jobid} -- #{body.inspect}"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
metadata
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: em-jack
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- dan sinclair
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-11-11 00:00:00 -06:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: eventmachine
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
version:
|
25
|
+
description: An evented Beanstalk client.
|
26
|
+
email: dj2@everburning.com
|
27
|
+
executables: []
|
28
|
+
|
29
|
+
extensions: []
|
30
|
+
|
31
|
+
extra_rdoc_files:
|
32
|
+
- README.rdoc
|
33
|
+
files:
|
34
|
+
- README.rdoc
|
35
|
+
- COPYING
|
36
|
+
- lib/em-jack.rb
|
37
|
+
- lib/em-jack/beanstalk_connection.rb
|
38
|
+
- lib/em-jack/connection.rb
|
39
|
+
- lib/em-jack/errors.rb
|
40
|
+
- lib/em-jack/job.rb
|
41
|
+
has_rdoc: true
|
42
|
+
homepage: http://github.com/dj2/em-jack/
|
43
|
+
licenses: []
|
44
|
+
|
45
|
+
post_install_message:
|
46
|
+
rdoc_options:
|
47
|
+
- --title
|
48
|
+
- EM-Jack Documentation
|
49
|
+
- --main
|
50
|
+
- README.rdoc
|
51
|
+
- --line-numbers
|
52
|
+
require_paths:
|
53
|
+
- lib
|
54
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
55
|
+
requirements:
|
56
|
+
- - ">="
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: "0"
|
59
|
+
version:
|
60
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
61
|
+
requirements:
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: "0"
|
65
|
+
version:
|
66
|
+
requirements: []
|
67
|
+
|
68
|
+
rubyforge_project:
|
69
|
+
rubygems_version: 1.3.5
|
70
|
+
signing_key:
|
71
|
+
specification_version: 3
|
72
|
+
summary: An evented Beanstalk client.
|
73
|
+
test_files: []
|
74
|
+
|