queue_classic 1.0.2 → 2.0.0rc1
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/queue_classic.rb +67 -8
- data/lib/queue_classic/conn.rb +97 -0
- data/lib/queue_classic/okjson.rb +15 -40
- data/lib/queue_classic/queries.rb +56 -0
- data/lib/queue_classic/queue.rb +14 -49
- data/lib/queue_classic/tasks.rb +25 -14
- data/lib/queue_classic/worker.rb +37 -23
- data/readme.md +520 -31
- data/test/helper.rb +21 -17
- data/test/queue_test.rb +29 -60
- data/test/worker_test.rb +57 -25
- metadata +10 -19
- data/lib/queue_classic/database.rb +0 -188
- data/lib/queue_classic/durable_array.rb +0 -51
- data/lib/queue_classic/job.rb +0 -42
- data/lib/queue_classic/logger.rb +0 -17
- data/test/database_helpers.rb +0 -13
- data/test/database_test.rb +0 -73
- data/test/durable_array_test.rb +0 -94
- data/test/job_test.rb +0 -82
data/lib/queue_classic.rb
CHANGED
@@ -6,18 +6,77 @@ require "uri"
|
|
6
6
|
$: << File.expand_path(__FILE__, "lib")
|
7
7
|
|
8
8
|
require "queue_classic/okjson"
|
9
|
-
require "queue_classic/
|
10
|
-
require "queue_classic/
|
11
|
-
require "queue_classic/worker"
|
12
|
-
require "queue_classic/logger"
|
9
|
+
require "queue_classic/conn"
|
10
|
+
require "queue_classic/queries"
|
13
11
|
require "queue_classic/queue"
|
14
|
-
require "queue_classic/
|
12
|
+
require "queue_classic/worker"
|
15
13
|
|
16
14
|
module QC
|
17
|
-
|
18
|
-
|
15
|
+
|
16
|
+
Root = File.expand_path(File.dirname(__FILE__))
|
17
|
+
SqlFunctions = File.join(QC::Root, "/sql/ddl.sql")
|
18
|
+
DropSqlFunctions = File.join(QC::Root, "/sql/drop_ddl.sql")
|
19
|
+
|
20
|
+
Log = Logger.new($stdout)
|
21
|
+
Log.level = (ENV["QC_LOG_LEVEL"] || Logger::DEBUG).to_i
|
22
|
+
Log.info("program=queue_classic log=true")
|
23
|
+
|
24
|
+
DB_URL =
|
25
|
+
ENV["QC_DATABASE_URL"] ||
|
26
|
+
ENV["DATABASE_URL"] ||
|
27
|
+
raise(ArgumentError, "missing QC_DATABASE_URL or DATABASE_URL")
|
28
|
+
|
29
|
+
# You can use the APP_NAME to query for
|
30
|
+
# postgres related process information in the
|
31
|
+
# pg_stat_activity table. Don't set this unless
|
32
|
+
# you are using PostgreSQL > 9.0
|
33
|
+
APP_NAME = ENV["QC_APP_NAME"]
|
34
|
+
|
35
|
+
# Why do you want to change the table name?
|
36
|
+
# Just deal with the default OK?
|
37
|
+
# If you do want to change this, you will
|
38
|
+
# need to update the PL/pgSQL lock_head() function.
|
39
|
+
# Come on. Don't do it.... Just stick with the default.
|
40
|
+
TABLE_NAME = "queue_classic_jobs"
|
41
|
+
|
42
|
+
# Each row in the table will have a column that
|
43
|
+
# notes the queue. You can point your workers
|
44
|
+
# at different queues but only one at a time.
|
45
|
+
QUEUE = ENV["QUEUE"] || "default"
|
46
|
+
|
47
|
+
# Set this to 1 for strict FIFO.
|
48
|
+
# There is nothing special about 9....
|
49
|
+
TOP_BOUND = (ENV["QC_TOP_BOUND"] || 9).to_i
|
50
|
+
|
51
|
+
# If you are using PostgreSQL > 9
|
52
|
+
# then you will have access to listen/notify with payload.
|
53
|
+
# Set this value if you wish to make your worker more efficient.
|
54
|
+
LISTENING_WORKER = !ENV["QC_LISTENING_WORKER"].nil?
|
55
|
+
|
56
|
+
# Set this variable if you wish for
|
57
|
+
# the worker to fork a UNIX process for
|
58
|
+
# each locked job. Remember to restablish
|
59
|
+
# any database connectoins. See the worker
|
60
|
+
# for more details.
|
61
|
+
FORK_WORKER = !ENV["QC_FORK_WORKER"].nil?
|
62
|
+
|
63
|
+
# The worker uses an exponential backoff
|
64
|
+
# algorithm to lock a job. This value will be used
|
65
|
+
# as the max exponent.
|
66
|
+
MAX_LOCK_ATTEMPTS = (ENV["QC_MAX_LOCK_ATTEMPTS"] || 5).to_i
|
67
|
+
|
68
|
+
if APP_NAME
|
69
|
+
Conn.execute("SET application_name = '#{APP_NAME}'")
|
70
|
+
end
|
19
71
|
|
20
72
|
def self.method_missing(sym, *args, &block)
|
21
|
-
|
73
|
+
default_queue.send(sym, *args, &block)
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.default_queue
|
77
|
+
@default_queue ||= begin
|
78
|
+
Queue.new(QUEUE, LISTENING_WORKER)
|
79
|
+
end
|
22
80
|
end
|
81
|
+
|
23
82
|
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module QC
|
2
|
+
module Conn
|
3
|
+
extend self
|
4
|
+
|
5
|
+
def execute(stmt, *params)
|
6
|
+
log("executing #{stmt.inspect}, #{params.inspect}")
|
7
|
+
begin
|
8
|
+
params = nil if params.empty?
|
9
|
+
r = connection.exec(stmt, params)
|
10
|
+
result = []
|
11
|
+
r.each {|t| result << t}
|
12
|
+
result.length > 1 ? result : result.pop
|
13
|
+
rescue PGError => e
|
14
|
+
log("execute exception=#{e.inspect}")
|
15
|
+
raise
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def notify(chan)
|
20
|
+
log("NOTIFY")
|
21
|
+
execute("NOTIFY #{chan}")
|
22
|
+
end
|
23
|
+
|
24
|
+
def listen(chan)
|
25
|
+
log("LISTEN")
|
26
|
+
execute("LISTEN #{chan}")
|
27
|
+
end
|
28
|
+
|
29
|
+
def unlisten(chan)
|
30
|
+
log("UNLISTEN")
|
31
|
+
execute("UNLISTEN #{chan}")
|
32
|
+
end
|
33
|
+
|
34
|
+
def drain_notify
|
35
|
+
until connection.notifies.nil?
|
36
|
+
log("draining notifications")
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def wait_for_notify(t)
|
41
|
+
log("waiting for notify timeout=#{t}")
|
42
|
+
connection.wait_for_notify(t) do |event, pid, msg|
|
43
|
+
log("received notification #{event}")
|
44
|
+
end
|
45
|
+
log("done waiting for notify")
|
46
|
+
end
|
47
|
+
|
48
|
+
def transaction
|
49
|
+
begin
|
50
|
+
execute("BEGIN")
|
51
|
+
yield
|
52
|
+
execute("COMMIT")
|
53
|
+
rescue Exception
|
54
|
+
execute("ROLLBACK")
|
55
|
+
raise
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def transaction_idle?
|
60
|
+
connection.transaction_status == PGconn::PQTRANS_IDLE
|
61
|
+
end
|
62
|
+
|
63
|
+
def connection
|
64
|
+
@connection ||= connect
|
65
|
+
end
|
66
|
+
|
67
|
+
def disconnect
|
68
|
+
connection.finish
|
69
|
+
@connection = nil
|
70
|
+
end
|
71
|
+
|
72
|
+
def connect
|
73
|
+
log("establishing connection")
|
74
|
+
conn = PGconn.connect(
|
75
|
+
db_url.host,
|
76
|
+
db_url.port || 5432,
|
77
|
+
nil, '', #opts, tty
|
78
|
+
db_url.path.gsub("/",""), # database name
|
79
|
+
db_url.user,
|
80
|
+
db_url.password
|
81
|
+
)
|
82
|
+
if conn.status != PGconn::CONNECTION_OK
|
83
|
+
log("connection error=#{conn.error}")
|
84
|
+
end
|
85
|
+
conn
|
86
|
+
end
|
87
|
+
|
88
|
+
def db_url
|
89
|
+
URI.parse(DB_URL)
|
90
|
+
end
|
91
|
+
|
92
|
+
def log(msg)
|
93
|
+
Log.info(msg)
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
end
|
data/lib/queue_classic/okjson.rb
CHANGED
@@ -1,7 +1,5 @@
|
|
1
1
|
module QC
|
2
|
-
#
|
3
|
-
#
|
4
|
-
# Copyright 2011, 2012 Keith Rarick
|
2
|
+
# Copyright 2011 Keith Rarick
|
5
3
|
#
|
6
4
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
7
5
|
# of this software and associated documentation files (the "Software"), to deal
|
@@ -263,12 +261,6 @@ module OkJson
|
|
263
261
|
def unquote(q)
|
264
262
|
q = q[1...-1]
|
265
263
|
a = q.dup # allocate a big enough string
|
266
|
-
rubydoesenc = false
|
267
|
-
# In ruby >= 1.9, a[w] is a codepoint, not a byte.
|
268
|
-
if a.class.method_defined?(:force_encoding)
|
269
|
-
a.force_encoding('UTF-8')
|
270
|
-
rubydoesenc = true
|
271
|
-
end
|
272
264
|
r, w = 0, 0
|
273
265
|
while r < q.length
|
274
266
|
c = q[r]
|
@@ -306,12 +298,7 @@ module OkJson
|
|
306
298
|
end
|
307
299
|
end
|
308
300
|
end
|
309
|
-
|
310
|
-
a[w] = '' << uchar
|
311
|
-
w += 1
|
312
|
-
else
|
313
|
-
w += ucharenc(a, w, uchar)
|
314
|
-
end
|
301
|
+
w += ucharenc(a, w, uchar)
|
315
302
|
else
|
316
303
|
raise Error, "invalid escape char #{q[r]} in \"#{q}\""
|
317
304
|
end
|
@@ -321,8 +308,6 @@ module OkJson
|
|
321
308
|
# Copy anything else byte-for-byte.
|
322
309
|
# Valid UTF-8 will remain valid UTF-8.
|
323
310
|
# Invalid UTF-8 will remain invalid UTF-8.
|
324
|
-
# In ruby >= 1.9, c is a codepoint, not a byte,
|
325
|
-
# in which case this is still what we want.
|
326
311
|
a[w] = c
|
327
312
|
r += 1
|
328
313
|
w += 1
|
@@ -457,10 +442,6 @@ module OkJson
|
|
457
442
|
t = StringIO.new
|
458
443
|
t.putc(?")
|
459
444
|
r = 0
|
460
|
-
|
461
|
-
# In ruby >= 1.9, s[r] is a codepoint, not a byte.
|
462
|
-
rubydoesenc = s.class.method_defined?(:encoding)
|
463
|
-
|
464
445
|
while r < s.length
|
465
446
|
case s[r]
|
466
447
|
when ?" then t.print('\\"')
|
@@ -475,13 +456,21 @@ module OkJson
|
|
475
456
|
case true
|
476
457
|
when Spc <= c && c <= ?~
|
477
458
|
t.putc(c)
|
478
|
-
when
|
479
|
-
u = c.ord
|
480
|
-
surrenc(t, u)
|
481
|
-
else
|
459
|
+
when true
|
482
460
|
u, size = uchardec(s, r)
|
483
461
|
r += size - 1 # we add one more at the bottom of the loop
|
484
|
-
|
462
|
+
if u < 0x10000
|
463
|
+
t.print('\\u')
|
464
|
+
hexenc4(t, u)
|
465
|
+
else
|
466
|
+
u1, u2 = unsubst(u)
|
467
|
+
t.print('\\u')
|
468
|
+
hexenc4(t, u1)
|
469
|
+
t.print('\\u')
|
470
|
+
hexenc4(t, u2)
|
471
|
+
end
|
472
|
+
else
|
473
|
+
# invalid byte; skip it
|
485
474
|
end
|
486
475
|
end
|
487
476
|
r += 1
|
@@ -491,20 +480,6 @@ module OkJson
|
|
491
480
|
end
|
492
481
|
|
493
482
|
|
494
|
-
def surrenc(t, u)
|
495
|
-
if u < 0x10000
|
496
|
-
t.print('\\u')
|
497
|
-
hexenc4(t, u)
|
498
|
-
else
|
499
|
-
u1, u2 = unsubst(u)
|
500
|
-
t.print('\\u')
|
501
|
-
hexenc4(t, u1)
|
502
|
-
t.print('\\u')
|
503
|
-
hexenc4(t, u2)
|
504
|
-
end
|
505
|
-
end
|
506
|
-
|
507
|
-
|
508
483
|
def hexenc4(t, u)
|
509
484
|
t.putc(Hex[(u>>12)&0xf])
|
510
485
|
t.putc(Hex[(u>>8)&0xf])
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module QC
|
2
|
+
module Queries
|
3
|
+
extend self
|
4
|
+
|
5
|
+
def insert(q_name, method, args, chan=nil)
|
6
|
+
s = "INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
|
7
|
+
res = Conn.execute(s, q_name, method, OkJson.encode(args))
|
8
|
+
Conn.notify(chan) if chan
|
9
|
+
end
|
10
|
+
|
11
|
+
def lock_head(q_name, top_bound)
|
12
|
+
s = "SELECT * FROM lock_head($1, $2)"
|
13
|
+
if r = Conn.execute(s, q_name, top_bound)
|
14
|
+
{
|
15
|
+
:id => r["id"],
|
16
|
+
:method => r["method"],
|
17
|
+
:args => OkJson.decode(r["args"])
|
18
|
+
}
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def count(q_name=nil)
|
23
|
+
s = "SELECT COUNT(*) FROM #{TABLE_NAME}"
|
24
|
+
s << " WHERE q_name = $1" if q_name
|
25
|
+
r = Conn.execute(*[s, q_name].compact)
|
26
|
+
r["count"].to_i
|
27
|
+
end
|
28
|
+
|
29
|
+
def delete(id)
|
30
|
+
Conn.execute("DELETE FROM #{TABLE_NAME} where id = $1", id)
|
31
|
+
end
|
32
|
+
|
33
|
+
def delete_all(q_name=nil)
|
34
|
+
s = "DELETE FROM #{TABLE_NAME}"
|
35
|
+
s << "WHERE q_name = $1" if q_name
|
36
|
+
Conn.execute(*[s, q_name].compact)
|
37
|
+
end
|
38
|
+
|
39
|
+
def load_functions
|
40
|
+
file = File.open(SqlFunctions)
|
41
|
+
Conn.transaction do
|
42
|
+
Conn.execute(file.read)
|
43
|
+
end
|
44
|
+
file.close
|
45
|
+
end
|
46
|
+
|
47
|
+
def drop_functions
|
48
|
+
file = File.open(DropSqlFunctions)
|
49
|
+
Conn.transaction do
|
50
|
+
Conn.execute(file.read)
|
51
|
+
end
|
52
|
+
file.close
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
data/lib/queue_classic/queue.rb
CHANGED
@@ -1,66 +1,31 @@
|
|
1
1
|
module QC
|
2
|
-
module AbstractQueue
|
3
|
-
|
4
|
-
def enqueue(job,*params)
|
5
|
-
if job.respond_to?(:signature) and job.respond_to?(:params)
|
6
|
-
params = *job.params
|
7
|
-
job = job.signature
|
8
|
-
end
|
9
|
-
array << {"job" => job, "params" => params}
|
10
|
-
end
|
11
|
-
|
12
|
-
def dequeue
|
13
|
-
array.first
|
14
|
-
end
|
15
|
-
|
16
|
-
def query(signature)
|
17
|
-
array.search_details_column(signature)
|
18
|
-
end
|
19
|
-
|
20
|
-
def delete(job)
|
21
|
-
array.delete(job)
|
22
|
-
end
|
23
|
-
|
24
|
-
def delete_all
|
25
|
-
array.each {|j| delete(j) }
|
26
|
-
end
|
27
|
-
|
28
|
-
def length
|
29
|
-
array.count
|
30
|
-
end
|
31
|
-
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
module QC
|
36
2
|
class Queue
|
37
3
|
|
38
|
-
|
39
|
-
extend AbstractQueue
|
4
|
+
attr_reader :name, :chan
|
40
5
|
|
41
|
-
def
|
42
|
-
|
6
|
+
def initialize(name, notify=false)
|
7
|
+
@name = name
|
8
|
+
@chan = @name if notify
|
43
9
|
end
|
44
10
|
|
45
|
-
def
|
46
|
-
|
11
|
+
def enqueue(method, *args)
|
12
|
+
Queries.insert(name, method, args, chan)
|
47
13
|
end
|
48
14
|
|
49
|
-
def
|
50
|
-
|
15
|
+
def lock(top_bound=TOP_BOUND)
|
16
|
+
Queries.lock_head(name, top_bound)
|
51
17
|
end
|
52
18
|
|
53
|
-
def
|
54
|
-
|
55
|
-
@array = DurableArray.new(@database)
|
19
|
+
def delete(id)
|
20
|
+
Queries.delete(id)
|
56
21
|
end
|
57
22
|
|
58
|
-
def
|
59
|
-
|
23
|
+
def delete_all(q_name=nil)
|
24
|
+
Queries.delete_all(q_name)
|
60
25
|
end
|
61
26
|
|
62
|
-
def
|
63
|
-
|
27
|
+
def count(q_name=nil)
|
28
|
+
Queries.count(q_name)
|
64
29
|
end
|
65
30
|
|
66
31
|
end
|
data/lib/queue_classic/tasks.rb
CHANGED
@@ -1,27 +1,38 @@
|
|
1
1
|
namespace :jobs do
|
2
|
-
|
3
|
-
|
4
|
-
task :work => 'qc:work'
|
5
|
-
|
2
|
+
desc "Alias for qc:work"
|
3
|
+
task :work => "qc:work"
|
6
4
|
end
|
7
5
|
|
8
6
|
namespace :qc do
|
9
|
-
|
10
|
-
desc 'Start a new worker for the (default or QUEUE) queue'
|
7
|
+
desc "Start a new worker for the (default or $QUEUE) queue"
|
11
8
|
task :work => :environment do
|
12
|
-
QC::Worker.new
|
9
|
+
QC::Worker.new(
|
10
|
+
QC::TABLE_NAME,
|
11
|
+
QC::TOP_BOUND,
|
12
|
+
QC::FORK_WORKER,
|
13
|
+
QC::LISTENING_WORKER,
|
14
|
+
QC::MAX_LOCK_ATTEMPTS
|
15
|
+
).start
|
13
16
|
end
|
14
17
|
|
15
|
-
desc
|
16
|
-
task :
|
17
|
-
puts QC::
|
18
|
+
desc "Returns the number of jobs in the (default or QUEUE) queue"
|
19
|
+
task :length => :environment do
|
20
|
+
puts QC::Worker.new(
|
21
|
+
QC::TABLE_NAME,
|
22
|
+
QC::TOP_BOUND,
|
23
|
+
QC::FORK_WORKER,
|
24
|
+
QC::LISTENING_WORKER,
|
25
|
+
QC::MAX_LOCK_ATTEMPTS
|
26
|
+
).length
|
18
27
|
end
|
19
28
|
|
20
|
-
desc
|
29
|
+
desc "Ensure the database has the necessary functions for QC"
|
21
30
|
task :load_functions => :environment do
|
22
|
-
|
23
|
-
db.load_functions
|
24
|
-
db.disconnect
|
31
|
+
QC::Queries.load_functions
|
25
32
|
end
|
26
33
|
|
34
|
+
desc "Remove queue_classic functions from database."
|
35
|
+
task :load_functions => :environment do
|
36
|
+
QC::Queries.drop_functions
|
37
|
+
end
|
27
38
|
end
|