queue_classic 0.3.2 → 0.3.3.pre
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/queue_classic.rb +2 -0
- data/lib/queue_classic/database.rb +19 -6
- data/lib/queue_classic/durable_array.rb +1 -11
- data/lib/queue_classic/worker.rb +43 -6
- data/test/queue_test.rb +1 -0
- metadata +44 -47
data/lib/queue_classic.rb
CHANGED
@@ -2,8 +2,10 @@ module QC
|
|
2
2
|
class Database
|
3
3
|
@@connection = nil
|
4
4
|
|
5
|
-
|
6
|
-
|
5
|
+
DATABASE_URL = (ENV["QC_DATABASE_URL"] || ENV["DATABASE_URL"])
|
6
|
+
MAX_TOP_BOUND = (ENV["QC_TOP_BOUND"] || 9).to_i
|
7
|
+
NOTIFY_TIMEOUT = (ENV["QC_NOTIFY_TIMEOUT"] || 10).to_i
|
8
|
+
DEFAULT_QUEUE_NAME = "queue_classic_jobs"
|
7
9
|
|
8
10
|
def self.create_queue(name)
|
9
11
|
db = new(name)
|
@@ -15,9 +17,9 @@ module QC
|
|
15
17
|
attr_reader :table_name
|
16
18
|
|
17
19
|
def initialize(queue_name=nil)
|
18
|
-
@top_boundry =
|
20
|
+
@top_boundry = MAX_TOP_BOUND
|
19
21
|
@table_name = queue_name || DEFAULT_QUEUE_NAME
|
20
|
-
@db_params = URI.parse(
|
22
|
+
@db_params = URI.parse(DATABASE_URL)
|
21
23
|
end
|
22
24
|
|
23
25
|
def init_db
|
@@ -26,6 +28,18 @@ module QC
|
|
26
28
|
load_functions
|
27
29
|
end
|
28
30
|
|
31
|
+
def listen
|
32
|
+
execute("LISTEN queue_classic_jobs")
|
33
|
+
end
|
34
|
+
|
35
|
+
def unlisten
|
36
|
+
execute("UNLISTEN queue_classic_jobs")
|
37
|
+
end
|
38
|
+
|
39
|
+
def wait_for_notify
|
40
|
+
connection.wait_for_notify(NOTIFY_TIMEOUT)
|
41
|
+
end
|
42
|
+
|
29
43
|
def waiting_conns
|
30
44
|
execute("SELECT * FROM pg_stat_activity WHERE datname = '#{@name}' AND waiting = 't' AND application_name = 'queue_classic'")
|
31
45
|
end
|
@@ -58,9 +72,8 @@ module QC
|
|
58
72
|
@db_params.user,
|
59
73
|
@db_params.password
|
60
74
|
)
|
61
|
-
@@connection.exec("LISTEN queue_classic_jobs")
|
62
75
|
@@connection.exec("SET application_name = 'queue_classic'")
|
63
|
-
silence_warnings unless
|
76
|
+
silence_warnings unless VERBOSE
|
64
77
|
end
|
65
78
|
@@connection
|
66
79
|
end
|
@@ -8,7 +8,6 @@ module QC
|
|
8
8
|
|
9
9
|
def <<(details)
|
10
10
|
execute("INSERT INTO #{@table_name} (details) VALUES ('#{details.to_json}')")
|
11
|
-
execute("NOTIFY queue_classic_jobs, 'new-job'")
|
12
11
|
end
|
13
12
|
|
14
13
|
def count
|
@@ -28,17 +27,8 @@ module QC
|
|
28
27
|
find_many { "SELECT * FROM #{@table_name} WHERE details LIKE '%#{q}%'" }
|
29
28
|
end
|
30
29
|
|
31
|
-
def lock_head
|
32
|
-
find_one { "SELECT * FROM lock_head('#{@table_name}')" }
|
33
|
-
end
|
34
|
-
|
35
30
|
def first
|
36
|
-
|
37
|
-
job
|
38
|
-
else
|
39
|
-
@database.connection.wait_for_notify(1) {|e,p,msg| job = lock_head if msg == "new-job" }
|
40
|
-
job
|
41
|
-
end
|
31
|
+
find_one { "SELECT * FROM lock_head('#{@table_name}')" }
|
42
32
|
end
|
43
33
|
|
44
34
|
def each
|
data/lib/queue_classic/worker.rb
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
module QC
|
2
2
|
class Worker
|
3
3
|
|
4
|
+
MAX_LOCK_ATTEMPTS = (ENV["QC_MAX_LOCK_ATTEMPTS"] || 5).to_i
|
5
|
+
|
4
6
|
def initialize
|
5
|
-
@running
|
6
|
-
@queue
|
7
|
+
@running = true
|
8
|
+
@queue = QC::Queue.new(ENV["QUEUE"])
|
9
|
+
@fork_worker = ENV["QC_FORK_WORKER"] == "true"
|
7
10
|
handle_signals
|
8
11
|
end
|
9
12
|
|
@@ -11,8 +14,12 @@ module QC
|
|
11
14
|
@running
|
12
15
|
end
|
13
16
|
|
17
|
+
def fork_worker?
|
18
|
+
@fork_worker
|
19
|
+
end
|
20
|
+
|
14
21
|
def handle_signals
|
15
|
-
%W(INT
|
22
|
+
%W(INT TERM).each do |sig|
|
16
23
|
trap(sig) do
|
17
24
|
if running?
|
18
25
|
@running = false
|
@@ -24,13 +31,22 @@ module QC
|
|
24
31
|
end
|
25
32
|
|
26
33
|
def start
|
27
|
-
while running?
|
28
|
-
|
34
|
+
while running?
|
35
|
+
if fork_worker?
|
36
|
+
fork_and_work
|
37
|
+
else
|
38
|
+
work
|
39
|
+
end
|
29
40
|
end
|
30
41
|
end
|
31
42
|
|
43
|
+
def fork_and_work
|
44
|
+
@cpid = fork { work }
|
45
|
+
Process.wait(@cpid)
|
46
|
+
end
|
47
|
+
|
32
48
|
def work
|
33
|
-
if job =
|
49
|
+
if job = lock_job
|
34
50
|
begin
|
35
51
|
job.work
|
36
52
|
rescue Object => e
|
@@ -39,6 +55,27 @@ module QC
|
|
39
55
|
@queue.delete(job)
|
40
56
|
end
|
41
57
|
end
|
58
|
+
job = nil
|
59
|
+
GC.start
|
60
|
+
end
|
61
|
+
|
62
|
+
def lock_job
|
63
|
+
attempts = 0
|
64
|
+
job = nil
|
65
|
+
until job
|
66
|
+
job = @queue.dequeue
|
67
|
+
if job.nil?
|
68
|
+
attempts += 1
|
69
|
+
if attempts < MAX_LOCK_ATTEMPTS
|
70
|
+
sleep(2**attempts)
|
71
|
+
next
|
72
|
+
else
|
73
|
+
break
|
74
|
+
end
|
75
|
+
else
|
76
|
+
end
|
77
|
+
end
|
78
|
+
job
|
42
79
|
end
|
43
80
|
|
44
81
|
#override this method to do whatever you want
|
data/test/queue_test.rb
CHANGED
metadata
CHANGED
@@ -1,49 +1,47 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: queue_classic
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.3.pre
|
5
|
+
prerelease: 6
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
7
|
+
authors:
|
8
8
|
- Ryan Smith
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
dependencies:
|
16
|
-
- !ruby/object:Gem::Dependency
|
12
|
+
date: 2011-08-22 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
17
15
|
name: pg
|
18
|
-
|
19
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
16
|
+
requirement: &2153629240 !ruby/object:Gem::Requirement
|
20
17
|
none: false
|
21
|
-
requirements:
|
22
|
-
- -
|
23
|
-
- !ruby/object:Gem::Version
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
24
21
|
version: 0.10.1
|
25
22
|
type: :runtime
|
26
|
-
version_requirements: *id001
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: json
|
29
23
|
prerelease: false
|
30
|
-
|
24
|
+
version_requirements: *2153629240
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: json
|
27
|
+
requirement: &2153628740 !ruby/object:Gem::Requirement
|
31
28
|
none: false
|
32
|
-
requirements:
|
33
|
-
- -
|
34
|
-
- !ruby/object:Gem::Version
|
35
|
-
version:
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
36
33
|
type: :runtime
|
37
|
-
|
38
|
-
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *2153628740
|
36
|
+
description: Queue Classic (beta) is a queueing library for Ruby apps (Rails, Sinatra,
|
37
|
+
Etc...) Queue Classic features asynchronous job polling, database maintained locks
|
38
|
+
and no ridiculous dependencies. As a matter of fact, Queue Classic only requires
|
39
|
+
the pg and json.
|
39
40
|
email: ryan@heroku.com
|
40
41
|
executables: []
|
41
|
-
|
42
42
|
extensions: []
|
43
|
-
|
44
43
|
extra_rdoc_files: []
|
45
|
-
|
46
|
-
files:
|
44
|
+
files:
|
47
45
|
- readme.md
|
48
46
|
- lib/queue_classic/database.rb
|
49
47
|
- lib/queue_classic/durable_array.rb
|
@@ -58,35 +56,34 @@ files:
|
|
58
56
|
- test/job_test.rb
|
59
57
|
- test/queue_test.rb
|
60
58
|
- test/worker_test.rb
|
61
|
-
has_rdoc: true
|
62
59
|
homepage: http://github.com/ryandotsmith/queue_classic
|
63
60
|
licenses: []
|
64
|
-
|
65
61
|
post_install_message:
|
66
62
|
rdoc_options: []
|
67
|
-
|
68
|
-
require_paths:
|
63
|
+
require_paths:
|
69
64
|
- lib
|
70
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
65
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
71
66
|
none: false
|
72
|
-
requirements:
|
73
|
-
- -
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version:
|
76
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - ! '>='
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
71
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
77
72
|
none: false
|
78
|
-
requirements:
|
79
|
-
- -
|
80
|
-
- !ruby/object:Gem::Version
|
81
|
-
version:
|
73
|
+
requirements:
|
74
|
+
- - ! '>'
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: 1.3.1
|
82
77
|
requirements: []
|
83
|
-
|
84
78
|
rubyforge_project:
|
85
|
-
rubygems_version: 1.
|
79
|
+
rubygems_version: 1.8.7
|
86
80
|
signing_key:
|
87
81
|
specification_version: 3
|
88
|
-
summary: Queue Classic (beta) is a queueing library for Ruby apps (Rails, Sinatra,
|
89
|
-
|
82
|
+
summary: Queue Classic (beta) is a queueing library for Ruby apps (Rails, Sinatra,
|
83
|
+
Etc...) Queue Classic features asynchronous job polling, database maintained locks
|
84
|
+
and no ridiculous dependencies. As a matter of fact, Queue Classic only requires
|
85
|
+
the pg and json.(simple)
|
86
|
+
test_files:
|
90
87
|
- test/durable_array_test.rb
|
91
88
|
- test/job_test.rb
|
92
89
|
- test/queue_test.rb
|