ironmq 1.1.1

Sign up to get free protection for your applications and to get access to all the features.
data/README.md ADDED
@@ -0,0 +1,18 @@
1
+ Beanstalk Ruby Client
2
+ =====================
3
+
4
+ Beanstalk is a simple, fast work queue. Its interface is generic, but was
5
+ originally designed for reducing the latency of page views in high-volume web
6
+ applications by running time-consuming tasks asynchronously.
7
+
8
+ For more information, see:
9
+
10
+ - <http://kr.github.com/beanstalkd/>
11
+ - <http://github.com/kr/beanstalkd/raw/master/doc/protocol.txt>
12
+
13
+ ## Contributors
14
+
15
+ - Isaac Feliu
16
+ - Peter Kieltyka
17
+ - Martyn Loughran
18
+ - Dustin Sallings
data/Rakefile ADDED
@@ -0,0 +1,36 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+
4
+ begin
5
+ require 'jeweler'
6
+ Jeweler::Tasks.new do |gem|
7
+ gem.name = "ironmq"
8
+ gem.summary = "Ruby client for beanstalkd"
9
+ gem.description = "Ruby client for beanstalkd"
10
+ gem.email = "kr@xph.us"
11
+ gem.homepage = "http://github.com/kr/beanstalk-client-ruby"
12
+ gem.authors = ["Keith Rarick"]
13
+ end
14
+ Jeweler::GemcutterTasks.new
15
+ rescue LoadError
16
+ puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
17
+ end
18
+
19
+ require 'rake/testtask'
20
+ Rake::TestTask.new(:test) do |test|
21
+ test.libs << 'lib' << 'test'
22
+ test.pattern = 'test/**/test_*.rb'
23
+ test.verbose = true
24
+ end
25
+
26
+ task :default => :test
27
+
28
+ require 'rake/rdoctask'
29
+ Rake::RDocTask.new do |rdoc|
30
+ version = File.exist?('VERSION') ? File.read('VERSION') : ""
31
+
32
+ rdoc.rdoc_dir = 'doc'
33
+ rdoc.title = "beanstalk-client #{version}"
34
+ rdoc.rdoc_files.include('README*')
35
+ rdoc.rdoc_files.include('lib/**/*.rb')
36
+ end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 1.1.1
data/ironmq.gemspec ADDED
@@ -0,0 +1,51 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE DIRECTLY
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
+ # -*- encoding: utf-8 -*-
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = %q{ironmq}
8
+ s.version = "1.1.1"
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = [%q{Keith Rarick}]
12
+ s.date = %q{2011-08-10}
13
+ s.description = %q{Ruby client for beanstalkd}
14
+ s.email = %q{kr@xph.us}
15
+ s.extra_rdoc_files = [
16
+ "LICENSE",
17
+ "README.md"
18
+ ]
19
+ s.files = [
20
+ "LICENSE",
21
+ "README.md",
22
+ "Rakefile",
23
+ "VERSION",
24
+ "ironmq.gemspec",
25
+ "lib/beanstalk-client.rb",
26
+ "lib/beanstalk-client/connection.rb",
27
+ "lib/beanstalk-client/errors.rb",
28
+ "lib/beanstalk-client/job.rb",
29
+ "lib/beanstalk-client/version.rb",
30
+ "test/helper.rb",
31
+ "test/test_beanstalk-client.rb",
32
+ "website/index.txt",
33
+ "website/javascripts/rounded_corners_lite.inc.js",
34
+ "website/stylesheets/screen.css",
35
+ "website/template.rhtml"
36
+ ]
37
+ s.homepage = %q{http://github.com/kr/beanstalk-client-ruby}
38
+ s.require_paths = [%q{lib}]
39
+ s.rubygems_version = %q{1.8.6}
40
+ s.summary = %q{Ruby client for beanstalkd}
41
+
42
+ if s.respond_to? :specification_version then
43
+ s.specification_version = 3
44
+
45
+ if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
46
+ else
47
+ end
48
+ else
49
+ end
50
+ end
51
+
@@ -0,0 +1,48 @@
1
+ # beanstalk-client.rb - client library for beanstalk
2
+
3
+ # Copyright (C) 2007 Philotic Inc.
4
+
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+
18
+ $:.unshift(File.dirname(__FILE__))
19
+
20
+ module Beanstalk
21
+ extend self
22
+
23
+ attr_accessor :select
24
+ end
25
+
26
+ require 'beanstalk-client/connection'
27
+ require 'beanstalk-client/errors'
28
+ require 'beanstalk-client/job'
29
+
30
+ # overrides for IronMQ
31
+
32
+ module Beanstalk
33
+ class Connection
34
+
35
+ def auth(token)
36
+ interact("auth #{token}\r\n",
37
+ %w(OK))
38
+ end
39
+
40
+ def read_job(word)
41
+ id, bytes = check_resp(word) #.map { |s| s.to_i }
42
+ bytes = bytes.to_i
43
+ body = read_bytes(bytes)
44
+ raise 'bad trailer' if read_bytes(2) != "\r\n"
45
+ [id, body, word == 'RESERVED']
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,456 @@
1
+ # beanstalk-client/connection.rb - client library for beanstalk
2
+
3
+ # Copyright (C) 2007 Philotic Inc.
4
+
5
+ # This program is free software: you can redistribute it and/or modify
6
+ # it under the terms of the GNU General Public License as published by
7
+ # the Free Software Foundation, either version 3 of the License, or
8
+ # (at your option) any later version.
9
+
10
+ # This program is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ # GNU General Public License for more details.
14
+
15
+ # You should have received a copy of the GNU General Public License
16
+ # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
+
18
+ require 'socket'
19
+ require 'fcntl'
20
+ require 'yaml'
21
+ require 'set'
22
+ require 'thread'
23
+
24
+ module Beanstalk
25
+ class Connection
26
+ attr_reader :addr
27
+
28
+ def initialize(addr, default_tube=nil)
29
+ @mutex = Mutex.new
30
+ @tube_mutex = Mutex.new
31
+ @waiting = false
32
+ @addr = addr
33
+ connect
34
+ @last_used = 'default'
35
+ @watch_list = [@last_used]
36
+ self.use(default_tube) if default_tube
37
+ self.watch(default_tube) if default_tube
38
+ end
39
+
40
+ def connect
41
+ host, port = addr.split(':')
42
+ @socket = TCPSocket.new(host, port.to_i)
43
+
44
+ # Don't leak fds when we exec.
45
+ @socket.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
46
+ end
47
+
48
+ def close
49
+ @socket.close
50
+ @socket = nil
51
+ end
52
+
53
+ def put(body, pri=65536, delay=0, ttr=120)
54
+ pri = pri.to_i
55
+ delay = delay.to_i
56
+ ttr = ttr.to_i
57
+ body = "#{body}" # Make sure that body.bytesize gives a useful number
58
+ interact("put #{pri} #{delay} #{ttr} #{body.bytesize}\r\n#{body}\r\n",
59
+ %w(INSERTED BURIED))[0].to_i
60
+ end
61
+
62
+ def yput(obj, pri=65536, delay=0, ttr=120)
63
+ put(YAML.dump(obj), pri, delay, ttr)
64
+ end
65
+
66
+ def peek_job(id)
67
+ interact("peek #{id}\r\n", :job)
68
+ end
69
+
70
+ def peek_ready()
71
+ interact("peek-ready\r\n", :job)
72
+ end
73
+
74
+ def peek_delayed()
75
+ interact("peek-delayed\r\n", :job)
76
+ end
77
+
78
+ def peek_buried()
79
+ interact("peek-buried\r\n", :job)
80
+ end
81
+
82
+ def on_tube(tube, &block)
83
+ @tube_mutex.lock
84
+ use tube
85
+ yield self
86
+ ensure
87
+ @tube_mutex.unlock
88
+ end
89
+
90
+ def reserve(timeout=nil)
91
+ raise WaitingForJobError if @waiting
92
+ @mutex.lock
93
+ if timeout.nil?
94
+ @socket.write("reserve\r\n")
95
+ else
96
+ @socket.write("reserve-with-timeout #{timeout}\r\n")
97
+ end
98
+
99
+ begin
100
+ @waiting = true
101
+ # Give the user a chance to select on multiple fds.
102
+ Beanstalk.select.call([@socket]) if Beanstalk.select
103
+ rescue WaitingForJobError
104
+ # just continue
105
+ ensure
106
+ @waiting = false
107
+ end
108
+
109
+ Job.new(self, *read_job('RESERVED'))
110
+ ensure
111
+ @mutex.unlock
112
+ end
113
+
114
+ def delete(id)
115
+ interact("delete #{id}\r\n", %w(DELETED))
116
+ :ok
117
+ end
118
+
119
+ def release(id, pri, delay)
120
+ id = id.to_i
121
+ pri = pri.to_i
122
+ delay = delay.to_i
123
+ interact("release #{id} #{pri} #{delay}\r\n", %w(RELEASED))
124
+ :ok
125
+ end
126
+
127
+ def bury(id, pri)
128
+ interact("bury #{id} #{pri}\r\n", %w(BURIED))
129
+ :ok
130
+ end
131
+
132
+ def touch(id)
133
+ interact("touch #{id}\r\n", %w(TOUCHED))
134
+ :ok
135
+ end
136
+
137
+ def kick(n)
138
+ interact("kick #{n}\r\n", %w(KICKED))[0].to_i
139
+ end
140
+
141
+ def use(tube)
142
+ return tube if tube == @last_used
143
+ @last_used = interact("use #{tube}\r\n", %w(USING))[0]
144
+ rescue BadFormatError
145
+ raise InvalidTubeName.new(tube)
146
+ end
147
+
148
+ def watch(tube)
149
+ return @watch_list.size if @watch_list.include?(tube)
150
+ r = interact("watch #{tube}\r\n", %w(WATCHING))[0].to_i
151
+ @watch_list += [tube]
152
+ return r
153
+ rescue BadFormatError
154
+ raise InvalidTubeName.new(tube)
155
+ end
156
+
157
+ def ignore(tube)
158
+ return @watch_list.size if !@watch_list.include?(tube)
159
+ r = interact("ignore #{tube}\r\n", %w(WATCHING))[0].to_i
160
+ @watch_list -= [tube]
161
+ return r
162
+ end
163
+
164
+ def stats()
165
+ interact("stats\r\n", :yaml)
166
+ end
167
+
168
+ def job_stats(id)
169
+ interact("stats-job #{id}\r\n", :yaml)
170
+ end
171
+
172
+ def stats_tube(tube)
173
+ interact("stats-tube #{tube}\r\n", :yaml)
174
+ end
175
+
176
+ def list_tubes()
177
+ interact("list-tubes\r\n", :yaml)
178
+ end
179
+
180
+ def list_tube_used()
181
+ interact("list-tube-used\r\n", %w(USING))[0]
182
+ end
183
+
184
+ def list_tubes_watched(cached=false)
185
+ return @watch_list if cached
186
+ @watch_list = interact("list-tubes-watched\r\n", :yaml)
187
+ end
188
+
189
+ private
190
+
191
+ def interact(cmd, rfmt)
192
+ raise WaitingForJobError if @waiting
193
+ @mutex.lock
194
+ @socket.write(cmd)
195
+ return read_yaml('OK') if rfmt == :yaml
196
+ return found_job if rfmt == :job
197
+ check_resp(*rfmt)
198
+ ensure
199
+ @mutex.unlock
200
+ end
201
+
202
+ def get_resp()
203
+ r = @socket.gets("\r\n")
204
+ raise EOFError if r == nil
205
+ r[0...-2]
206
+ end
207
+
208
+ def check_resp(*words)
209
+ r = get_resp()
210
+ rword, *vals = r.split(/\s+/)
211
+ if (words.size > 0) and !words.include?(rword)
212
+ raise UnexpectedResponse.classify(rword, r)
213
+ end
214
+ vals
215
+ end
216
+
217
+ def found_job()
218
+ Job.new(self, *read_job('FOUND'))
219
+ rescue NotFoundError
220
+ nil
221
+ end
222
+
223
+ def read_job(word)
224
+ id, bytes = check_resp(word).map{|s| s.to_i}
225
+ body = read_bytes(bytes)
226
+ raise 'bad trailer' if read_bytes(2) != "\r\n"
227
+ [id, body, word == 'RESERVED']
228
+ end
229
+
230
+ def read_yaml(word)
231
+ bytes_s, = check_resp(word)
232
+ yaml = read_bytes(bytes_s.to_i)
233
+ raise 'bad trailer' if read_bytes(2) != "\r\n"
234
+ YAML::load(yaml)
235
+ end
236
+
237
+ def read_bytes(n)
238
+ str = @socket.read(n)
239
+ raise EOFError, 'End of file reached' if str == nil
240
+ raise EOFError, 'End of file reached' if str.size < n
241
+ str
242
+ end
243
+ end
244
+
245
+ class Pool
246
+ attr_accessor :last_conn
247
+
248
+ def initialize(addrs, default_tube=nil)
249
+ @addrs = addrs
250
+ @watch_list = ['default']
251
+ @default_tube=default_tube
252
+ @watch_list = [default_tube] if default_tube
253
+ connect()
254
+ end
255
+
256
+ def connect()
257
+ @connections ||= {}
258
+ @addrs.each do |addr|
259
+ begin
260
+ if !@connections.include?(addr)
261
+ @connections[addr] = Connection.new(addr, @default_tube)
262
+ prev_watched = @connections[addr].list_tubes_watched()
263
+ to_ignore = prev_watched - @watch_list
264
+ @watch_list.each{|tube| @connections[addr].watch(tube)}
265
+ to_ignore.each{|tube| @connections[addr].ignore(tube)}
266
+ end
267
+ rescue Errno::ECONNREFUSED
268
+ raise NotConnected
269
+ rescue Exception => ex
270
+ puts "#{ex.class}: #{ex}"
271
+ end
272
+ end
273
+ @connections.size
274
+ end
275
+
276
+ def open_connections()
277
+ @connections.values()
278
+ end
279
+
280
+ def last_server
281
+ @last_conn.addr
282
+ end
283
+
284
+ def auth(token)
285
+ send_to_all_conns(:auth, token)
286
+ end
287
+
288
+ # Put a job on the queue.
289
+ #
290
+ # == Parameters:
291
+ #
292
+ # * <tt>body</tt>: the payload of the job (use Beanstalk::Pool#yput / Beanstalk::Job#ybody to automatically serialize your payload with YAML)
293
+ # * <tt>pri</tt>: priority. Default 65536 (higher numbers are higher priority)
294
+ # * <tt>delay</tt>: how long to wait until making the job available for reservation
295
+ # * <tt>ttr</tt>: time in seconds for the job to reappear on the queue (if beanstalk doesn't hear from a consumer within this time, assume the consumer died and make the job available for someone else to process). Default 120 seconds.
296
+ def put(body, pri=65536, delay=0, ttr=120)
297
+ send_to_rand_conn(:put, body, pri, delay, ttr)
298
+ end
299
+
300
+ # Like put, but serialize the object with YAML.
301
+ def yput(obj, pri=65536, delay=0, ttr=120)
302
+ send_to_rand_conn(:yput, obj, pri, delay, ttr)
303
+ end
304
+
305
+ def on_tube(tube, &block)
306
+ send_to_rand_conn(:on_tube, tube, &block)
307
+ end
308
+
309
+ # Reserve a job from the queue.
310
+ #
311
+ # == Parameters
312
+ #
313
+ # * <tt>timeout</tt> - Time (in seconds) to wait for a job to be available. If nil, wait indefinitely.
314
+ def reserve(timeout=nil)
315
+ send_to_rand_conn(:reserve, timeout)
316
+ end
317
+
318
+ def use(tube)
319
+ send_to_all_conns(:use, tube)
320
+ end
321
+
322
+ def watch(tube)
323
+ r = send_to_all_conns(:watch, tube)
324
+ @watch_list = send_to_rand_conn(:list_tubes_watched, true)
325
+ return r
326
+ end
327
+
328
+ def ignore(tube)
329
+ r = send_to_all_conns(:ignore, tube)
330
+ @watch_list = send_to_rand_conn(:list_tubes_watched, true)
331
+ return r
332
+ end
333
+
334
+ def raw_stats()
335
+ send_to_all_conns(:stats)
336
+ end
337
+
338
+ def stats()
339
+ sum_hashes(raw_stats.values)
340
+ end
341
+
342
+ def raw_stats_tube(tube)
343
+ send_to_all_conns(:stats_tube, tube)
344
+ end
345
+
346
+ def stats_tube(tube)
347
+ sum_hashes(raw_stats_tube(tube).values)
348
+ end
349
+
350
+ def list_tubes()
351
+ send_to_all_conns(:list_tubes)
352
+ end
353
+
354
+ def list_tube_used()
355
+ send_to_all_conns(:list_tube_used)
356
+ end
357
+
358
+ def list_tubes_watched(*args)
359
+ send_to_all_conns(:list_tubes_watched, *args)
360
+ end
361
+
362
+ def remove(conn)
363
+ connection = @connections.delete(conn.addr)
364
+ connection.close if connection
365
+ connection
366
+ end
367
+
368
+ # Close all open connections for this pool
369
+ def close
370
+ while @connections.size > 0
371
+ addr = @connections.keys.last
372
+ conn = @connections[addr]
373
+ @connections.delete(addr)
374
+ conn.close
375
+ end
376
+ end
377
+
378
+ def peek_ready()
379
+ send_to_each_conn_first_res(:peek_ready)
380
+ end
381
+
382
+ def peek_delayed()
383
+ send_to_each_conn_first_res(:peek_delayed)
384
+ end
385
+
386
+ def peek_buried()
387
+ send_to_each_conn_first_res(:peek_buried)
388
+ end
389
+
390
+ def peek_job(id)
391
+ make_hash(send_to_all_conns(:peek_job, id))
392
+ end
393
+
394
+ private
395
+
396
+ def call_wrap(c, *args, &block)
397
+ self.last_conn = c
398
+ c.send(*args, &block)
399
+ rescue UnexpectedResponse => ex
400
+ raise ex
401
+ rescue EOFError, Errno::ECONNRESET, Errno::EPIPE => ex
402
+ self.remove(c)
403
+ raise ex
404
+ end
405
+
406
+ def retry_wrap(*args)
407
+ yield
408
+ rescue DrainingError
409
+ # Don't reconnect -- we're not interested in this server
410
+ retry
411
+ rescue EOFError, Errno::ECONNRESET, Errno::EPIPE
412
+ connect()
413
+ retry
414
+ end
415
+
416
+ def send_to_each_conn_first_res(*args)
417
+ connect()
418
+ retry_wrap{open_connections.inject(nil) {|r,c| r or call_wrap(c, *args)}}
419
+ end
420
+
421
+ def send_to_rand_conn(*args, &block)
422
+ connect()
423
+ retry_wrap{call_wrap(pick_connection, *args, &block)}
424
+ end
425
+
426
+ def send_to_all_conns(*args)
427
+ connect()
428
+ retry_wrap{compact_hash(map_hash(@connections){|c| call_wrap(c, *args)})}
429
+ end
430
+
431
+ def pick_connection()
432
+ open_connections[rand(open_connections.size)] or raise NotConnected
433
+ end
434
+
435
+ def make_hash(pairs)
436
+ Hash[*pairs.inject([]){|a,b|a+b}]
437
+ end
438
+
439
+ def map_hash(h)
440
+ make_hash(h.map{|k,v| [k, yield(v)]})
441
+ end
442
+
443
+ def compact_hash(hash)
444
+ hash.reject{|k,v| v == nil}
445
+ end
446
+
447
+ def sum_hashes(hs)
448
+ hs.inject({}){|a,b| a.merge(b) {|k,o,n| combine_stats(k, o, n)}}
449
+ end
450
+
451
+ DONT_ADD = Set['name', 'version', 'pid']
452
+ def combine_stats(k, a, b)
453
+ DONT_ADD.include?(k) ? Set[a] + Set[b] : a + b
454
+ end
455
+ end
456
+ end