em-jack 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/COPYING +21 -0
- data/README.rdoc +80 -0
- data/lib/em-jack.rb +12 -0
- data/lib/em-jack/beanstalk_connection.rb +30 -0
- data/lib/em-jack/connection.rb +231 -0
- data/lib/em-jack/errors.rb +7 -0
- data/lib/em-jack/job.rb +23 -0
- metadata +74 -0
data/COPYING
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
Copyright (c) 2009, dan sinclair. All rights reserved.
|
2
|
+
|
3
|
+
Redistribution and use in source and binary forms, with or without
|
4
|
+
modification, are permitted provided that the following conditions are
|
5
|
+
met:
|
6
|
+
* Redistributions of source code must retain the above copyright
|
7
|
+
notice, this list of conditions and the following disclaimer.
|
8
|
+
* Redistributions in binary form must reproduce the above copyright
|
9
|
+
notice, this list of conditions and the following disclaimer in the
|
10
|
+
documentation and/or other materials provided with the distribution.
|
11
|
+
|
12
|
+
THIS SOFTWARE IS PROVIDED BY dan sinclair ''AS IS'' AND ANY
|
13
|
+
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
14
|
+
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
15
|
+
DISCLAIMED. IN NO EVENT SHALL dan sinclair BE LIABLE FOR ANY
|
16
|
+
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
17
|
+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
18
|
+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
19
|
+
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
20
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
21
|
+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
= EMJack
|
2
|
+
An attempt to wrap portions of the Beanstalk protocol with EventMachine. Every command
|
3
|
+
will return a deferrable object. That object will succeed or fail depending on the
|
4
|
+
reply returned from Beanstalk.
|
5
|
+
|
6
|
+
The current, default, errback is to print the error message received.
|
7
|
+
|
8
|
+
One thing to keep in mind. The Beanstalk protocol executes all commands serially.
|
9
|
+
So, if you send a reserve command and there are no jobs Beanstalk _won't_ process
|
10
|
+
any of the commands that come after the reserve until the reserve completes.
|
11
|
+
|
12
|
+
This is a bit of a gotcha when sending a series of reserve, delete, etc and there
|
13
|
+
are no available jobs.
|
14
|
+
|
15
|
+
= Dependencies
|
16
|
+
- EventMachine
|
17
|
+
- RSpec (to run the tests)
|
18
|
+
|
19
|
+
= Examples
|
20
|
+
EM.run {
|
21
|
+
jack = EMJack::Connection.new
|
22
|
+
|
23
|
+
r = jack.use('mytube')
|
24
|
+
r.callback { |tube| puts "Using #{tube}" }
|
25
|
+
|
26
|
+
r = jack.reserve
|
27
|
+
r.callback do |job|
|
28
|
+
puts job.jobid
|
29
|
+
process(job)
|
30
|
+
|
31
|
+
r2 = jack.delete(job)
|
32
|
+
r2.callback { puts "Successfully deleted" }
|
33
|
+
end
|
34
|
+
|
35
|
+
r = jack.put("my message", :ttr => 300)
|
36
|
+
r.callback { |jobid| puts "put successful #{jobid}" }
|
37
|
+
|
38
|
+
r = jack.stats
|
39
|
+
r.callback { |stats| puts "Server up for #{stats['uptime']} seconds" "}
|
40
|
+
|
41
|
+
r = jack.stats(:tube, "mytube")
|
42
|
+
r.callback { |stats| puts "Total jobs #{stats['total-jobs']}" }
|
43
|
+
|
44
|
+
r = jack.list(:tubes)
|
45
|
+
r.callback { |tubes| puts "There are #{tubes.length} tubes defined" }
|
46
|
+
}
|
47
|
+
|
48
|
+
|
49
|
+
EMJack#each_job is useful for scenarios where the client is a job worker
|
50
|
+
and intended to process jobs continuously as they become available. Once
|
51
|
+
the queue is empty, the client will block and wait for a new job.
|
52
|
+
|
53
|
+
If multiple workers connect to the queue Beanstalkd will round-robin between
|
54
|
+
the workers.
|
55
|
+
EM.run {
|
56
|
+
jack = EMJack::Connection.new
|
57
|
+
|
58
|
+
jack.each_job do |job|
|
59
|
+
puts "Got job ##{job.jobid}: #{job}"
|
60
|
+
|
61
|
+
if process(job)
|
62
|
+
r = jack.delete(job)
|
63
|
+
r.callback { puts "*Deleted #{job}*" }
|
64
|
+
else
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def process(job)
|
69
|
+
# Some kind of job processing
|
70
|
+
end
|
71
|
+
}
|
72
|
+
|
73
|
+
|
74
|
+
= Contact
|
75
|
+
If you've got any questions, comments or bugs, please let me know. You can
|
76
|
+
contact me by email at dj2 at everburning dot com.
|
77
|
+
|
78
|
+
|
79
|
+
= Contributors
|
80
|
+
Peter Kieltyka (EMJack#each_job)
|
data/lib/em-jack.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
|
3
|
+
module EMJack
|
4
|
+
class BeanstalkConnection < EM::Connection
|
5
|
+
attr_accessor :client
|
6
|
+
|
7
|
+
def connection_completed
|
8
|
+
@client.connected
|
9
|
+
end
|
10
|
+
|
11
|
+
def receive_data(data)
|
12
|
+
@client.received(data)
|
13
|
+
end
|
14
|
+
|
15
|
+
def send(command, *args)
|
16
|
+
cmd = command.to_s
|
17
|
+
cmd << " #{args.join(" ")}" unless args.length == 0
|
18
|
+
cmd << "\r\n"
|
19
|
+
send_data(cmd)
|
20
|
+
end
|
21
|
+
|
22
|
+
def send_with_data(command, data, *args)
|
23
|
+
send_data("#{command.to_s} #{args.join(" ")}\r\n#{data}\r\n")
|
24
|
+
end
|
25
|
+
|
26
|
+
def unbind
|
27
|
+
@client.disconnected
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,231 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module EMJack
|
5
|
+
class Connection
|
6
|
+
RETRY_COUNT = 5
|
7
|
+
|
8
|
+
attr_accessor :host, :port
|
9
|
+
|
10
|
+
def initialize(opts = {})
|
11
|
+
@host = opts[:host] || 'localhost'
|
12
|
+
@port = opts[:port] || 11300
|
13
|
+
@tube = opts[:tube]
|
14
|
+
|
15
|
+
@used_tube = 'default'
|
16
|
+
@watched_tubes = ['default']
|
17
|
+
|
18
|
+
@data = ""
|
19
|
+
@retries = 0
|
20
|
+
@in_reserve = false
|
21
|
+
@deferrables = []
|
22
|
+
|
23
|
+
@conn = EM::connect(host, port, EMJack::BeanstalkConnection) do |conn|
|
24
|
+
conn.client = self
|
25
|
+
end
|
26
|
+
|
27
|
+
unless @tube.nil?
|
28
|
+
use(@tube)
|
29
|
+
watch(@tube)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def use(tube)
|
34
|
+
return if @used_tube == tube
|
35
|
+
@used_tube = tube
|
36
|
+
@conn.send(:use, tube)
|
37
|
+
add_deferrable
|
38
|
+
end
|
39
|
+
|
40
|
+
def watch(tube)
|
41
|
+
return if @watched_tubes.include?(tube)
|
42
|
+
@watched_tubes.push(tube)
|
43
|
+
@conn.send(:watch, tube)
|
44
|
+
add_deferrable
|
45
|
+
end
|
46
|
+
|
47
|
+
def ignore(tube)
|
48
|
+
return if not @watched_tubes.include?(tube)
|
49
|
+
@watched_tubes.delete(tube)
|
50
|
+
@conn.send(:ignore, tube)
|
51
|
+
add_deferrable
|
52
|
+
end
|
53
|
+
|
54
|
+
def reserve(timeout = nil)
|
55
|
+
if timeout
|
56
|
+
@conn.send(:'reserve-with-timeout', timeout)
|
57
|
+
else
|
58
|
+
@conn.send(:reserve)
|
59
|
+
end
|
60
|
+
add_deferrable
|
61
|
+
end
|
62
|
+
|
63
|
+
def each_job(&block)
|
64
|
+
work = Proc.new do
|
65
|
+
r = reserve
|
66
|
+
r.callback do |job|
|
67
|
+
block.call(job)
|
68
|
+
EM.next_tick { work.call }
|
69
|
+
end
|
70
|
+
end
|
71
|
+
work.call
|
72
|
+
end
|
73
|
+
|
74
|
+
def stats(type = nil, val = nil)
|
75
|
+
case(type)
|
76
|
+
when nil then @conn.send(:stats)
|
77
|
+
when :tube then @conn.send(:'stats-tube', val)
|
78
|
+
when :job then @conn.send(:'stats-job', val.jobid)
|
79
|
+
else raise EMJack::InvalidCommand.new
|
80
|
+
end
|
81
|
+
add_deferrable
|
82
|
+
end
|
83
|
+
|
84
|
+
def list(type = nil)
|
85
|
+
case(type)
|
86
|
+
when nil then @conn.send(:'list-tubes')
|
87
|
+
when :used then @conn.send(:'list-tube-used')
|
88
|
+
when :watched then @conn.send(:'list-tubes-watched')
|
89
|
+
else raise EMJack::InvalidCommand.new
|
90
|
+
end
|
91
|
+
add_deferrable
|
92
|
+
end
|
93
|
+
|
94
|
+
def delete(job)
|
95
|
+
return if job.nil?
|
96
|
+
@conn.send(:delete, job.jobid)
|
97
|
+
add_deferrable
|
98
|
+
end
|
99
|
+
|
100
|
+
def release(job)
|
101
|
+
return if job.nil?
|
102
|
+
@conn.send(:release, job.jobid, 0, 0)
|
103
|
+
add_deferrable
|
104
|
+
end
|
105
|
+
|
106
|
+
def put(msg, opts = {})
|
107
|
+
pri = (opts[:priority] || 65536).to_i
|
108
|
+
if pri< 0
|
109
|
+
pri = 65536
|
110
|
+
elsif pri > (2 ** 32)
|
111
|
+
pri = 2 ** 32
|
112
|
+
end
|
113
|
+
|
114
|
+
delay = (opts[:delay] || 0).to_i
|
115
|
+
delay = 0 if delay < 0
|
116
|
+
|
117
|
+
ttr = (opts[:ttr] || 300).to_i
|
118
|
+
ttr = 300 if ttr < 0
|
119
|
+
|
120
|
+
m = msg.to_s
|
121
|
+
|
122
|
+
@conn.send_with_data(:put, m, pri, delay, ttr, m.length)
|
123
|
+
add_deferrable
|
124
|
+
end
|
125
|
+
|
126
|
+
def connected
|
127
|
+
@retries = 0
|
128
|
+
end
|
129
|
+
|
130
|
+
def disconnected
|
131
|
+
# XXX I think I need to run out the deferrables as failed here
|
132
|
+
# since the connection was dropped
|
133
|
+
|
134
|
+
raise EMJack::Disconnected if @retries >= RETRY_COUNT
|
135
|
+
@retries += 1
|
136
|
+
EM.add_timer(1) { @conn.reconnect(@host, @port) }
|
137
|
+
end
|
138
|
+
|
139
|
+
def add_deferrable
|
140
|
+
df = EM::DefaultDeferrable.new
|
141
|
+
df.errback { |err| puts "ERROR: #{err}" }
|
142
|
+
|
143
|
+
@deferrables.push(df)
|
144
|
+
df
|
145
|
+
end
|
146
|
+
|
147
|
+
def received(data)
|
148
|
+
@data << data
|
149
|
+
|
150
|
+
until @data.empty?
|
151
|
+
idx = @data.index(/\r\n/)
|
152
|
+
break if idx.nil?
|
153
|
+
|
154
|
+
first = @data[0..(idx + 1)]
|
155
|
+
|
156
|
+
handled = false
|
157
|
+
%w(OUT_OF_MEMORY INTERNAL_ERROR DRAINING BAD_FORMAT
|
158
|
+
UNKNOWN_COMMAND EXPECTED_CRLF JOB_TOO_BIG DEADLINE_SOON
|
159
|
+
TIMED_OUT NOT_FOUND).each do |cmd|
|
160
|
+
next unless first =~ /^#{cmd}\r\n/i
|
161
|
+
df = @deferrables.shift
|
162
|
+
df.fail(cmd.downcase.to_sym)
|
163
|
+
|
164
|
+
@data = @data[(cmd.length + 2)..-1]
|
165
|
+
handled = true
|
166
|
+
break
|
167
|
+
end
|
168
|
+
next if handled
|
169
|
+
|
170
|
+
case (first)
|
171
|
+
when /^DELETED\r\n/ then
|
172
|
+
df = @deferrables.shift
|
173
|
+
df.succeed
|
174
|
+
|
175
|
+
when /^INSERTED\s+(\d+)\r\n/ then
|
176
|
+
df = @deferrables.shift
|
177
|
+
df.succeed($1.to_i)
|
178
|
+
|
179
|
+
when /^BURIED\s+(\d+)\r\n/ then
|
180
|
+
df = @deferrables.shift
|
181
|
+
df.fail(:buried, $1.to_i)
|
182
|
+
|
183
|
+
when /^USING\s+(.*)\r\n/ then
|
184
|
+
df = @deferrables.shift
|
185
|
+
df.succeed($1)
|
186
|
+
|
187
|
+
when /^WATCHING\s+(\d+)\r\n/ then
|
188
|
+
df = @deferrables.shift
|
189
|
+
df.succeed($1.to_i)
|
190
|
+
|
191
|
+
when /^OK\s+(\d+)\r\n/ then
|
192
|
+
bytes = $1.to_i
|
193
|
+
|
194
|
+
body, @data = extract_body(bytes, @data)
|
195
|
+
break if body.nil?
|
196
|
+
|
197
|
+
df = @deferrables.shift
|
198
|
+
df.succeed(YAML.load(body))
|
199
|
+
next
|
200
|
+
|
201
|
+
when /^RESERVED\s+(\d+)\s+(\d+)\r\n/ then
|
202
|
+
id = $1.to_i
|
203
|
+
bytes = $2.to_i
|
204
|
+
|
205
|
+
body, @data = extract_body(bytes, @data)
|
206
|
+
break if body.nil?
|
207
|
+
|
208
|
+
df = @deferrables.shift
|
209
|
+
job = EMJack::Job.new(self, id, body)
|
210
|
+
df.succeed(job)
|
211
|
+
next
|
212
|
+
|
213
|
+
else
|
214
|
+
break
|
215
|
+
end
|
216
|
+
|
217
|
+
@data = @data[(@data.index(/\r\n/) + 2)..-1]
|
218
|
+
@data = "" if @data.nil?
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def extract_body(bytes, data)
|
223
|
+
rem = data[(data.index(/\r\n/) + 2)..-1]
|
224
|
+
return [nil, data] if rem.length < bytes
|
225
|
+
body = rem[0..(bytes - 1)]
|
226
|
+
data = rem[(bytes + 2)..-1]
|
227
|
+
data = "" if data.nil?
|
228
|
+
[body, data]
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
data/lib/em-jack/job.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
module EMJack
|
2
|
+
class Job
|
3
|
+
attr_accessor :jobid, :body, :ttr, :conn
|
4
|
+
|
5
|
+
def initialize(conn, jobid, body)
|
6
|
+
@conn = conn
|
7
|
+
@jobid = jobid.to_i
|
8
|
+
@body = body
|
9
|
+
end
|
10
|
+
|
11
|
+
def delete
|
12
|
+
@conn.delete(self)
|
13
|
+
end
|
14
|
+
|
15
|
+
def stats
|
16
|
+
@conn.stats(:job, self)
|
17
|
+
end
|
18
|
+
|
19
|
+
def to_s
|
20
|
+
"#{@jobid} -- #{body.inspect}"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
metadata
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: em-jack
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- dan sinclair
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-11-11 00:00:00 -06:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: eventmachine
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
version:
|
25
|
+
description: An evented Beanstalk client.
|
26
|
+
email: dj2@everburning.com
|
27
|
+
executables: []
|
28
|
+
|
29
|
+
extensions: []
|
30
|
+
|
31
|
+
extra_rdoc_files:
|
32
|
+
- README.rdoc
|
33
|
+
files:
|
34
|
+
- README.rdoc
|
35
|
+
- COPYING
|
36
|
+
- lib/em-jack.rb
|
37
|
+
- lib/em-jack/beanstalk_connection.rb
|
38
|
+
- lib/em-jack/connection.rb
|
39
|
+
- lib/em-jack/errors.rb
|
40
|
+
- lib/em-jack/job.rb
|
41
|
+
has_rdoc: true
|
42
|
+
homepage: http://github.com/dj2/em-jack/
|
43
|
+
licenses: []
|
44
|
+
|
45
|
+
post_install_message:
|
46
|
+
rdoc_options:
|
47
|
+
- --title
|
48
|
+
- EM-Jack Documentation
|
49
|
+
- --main
|
50
|
+
- README.rdoc
|
51
|
+
- --line-numbers
|
52
|
+
require_paths:
|
53
|
+
- lib
|
54
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
55
|
+
requirements:
|
56
|
+
- - ">="
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
version: "0"
|
59
|
+
version:
|
60
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
61
|
+
requirements:
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: "0"
|
65
|
+
version:
|
66
|
+
requirements: []
|
67
|
+
|
68
|
+
rubyforge_project:
|
69
|
+
rubygems_version: 1.3.5
|
70
|
+
signing_key:
|
71
|
+
specification_version: 3
|
72
|
+
summary: An evented Beanstalk client.
|
73
|
+
test_files: []
|
74
|
+
|