pg_queue 0.0.0a → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.rspec +1 -0
- data/lib/pg_queue/job.rb +6 -5
- data/lib/pg_queue/pg_extensions.rb +41 -0
- data/lib/pg_queue/version.rb +1 -1
- data/lib/pg_queue/worker.rb +12 -30
- data/lib/pg_queue.rb +38 -1
- data/pg_queue.gemspec +3 -0
- data/spec/pg_queue/job_spec.rb +22 -0
- data/spec/pg_queue/worker_spec.rb +54 -0
- data/spec/pg_queue_spec.rb +65 -0
- data/spec/spec_helper.rb +3 -0
- metadata +40 -9
- data/lib/pg_queue/queue.rb +0 -26
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/lib/pg_queue/job.rb
CHANGED
@@ -1,18 +1,19 @@
|
|
1
|
+
require "multi_json"
|
2
|
+
|
1
3
|
module PgQueue
|
2
4
|
class Job
|
3
5
|
attr_reader :id, :klass, :args
|
4
6
|
|
5
7
|
def initialize(attributes)
|
6
8
|
@id = attributes["id"]
|
7
|
-
|
8
|
-
@
|
9
|
-
@args = JSON.load(attributes["args"])
|
9
|
+
@klass = Object.const_get(attributes["class_name"])
|
10
|
+
@args = MultiJson.decode(attributes["args"])
|
10
11
|
end
|
11
12
|
|
12
13
|
def perform
|
13
|
-
|
14
|
+
PgQueue.logger.debug("performing job #{@id}")
|
14
15
|
klass.perform(*args)
|
15
|
-
|
16
|
+
PgQueue.logger.debug("job #{@id} performed")
|
16
17
|
end
|
17
18
|
end
|
18
19
|
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module PgQueue
|
2
|
+
module PgExtensions
|
3
|
+
def insert(table_name, hash, returning = nil)
|
4
|
+
fields = hash.keys.join(", ")
|
5
|
+
variables = hash.size.times.map { |n| "$#{n+1}" }.join(", ")
|
6
|
+
sql = "INSERT INTO #{table_name} (#{fields}) VALUES (#{variables})"
|
7
|
+
if returning
|
8
|
+
sql << " RETURNING #{returning}"
|
9
|
+
exec(sql, hash.values).getvalue(0, 0)
|
10
|
+
else
|
11
|
+
exec(sql, hash.values)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def delete(table_name, where)
|
16
|
+
i = 0
|
17
|
+
conditions = where.map { |k, v| i += 1; "#{k} = $#{i}" }.join(" AND ")
|
18
|
+
exec("DELETE FROM #{table_name} WHERE #{conditions}", where.values)
|
19
|
+
end
|
20
|
+
|
21
|
+
def first(sql)
|
22
|
+
result = exec(sql)
|
23
|
+
return nil unless result.count > 0
|
24
|
+
result[0]
|
25
|
+
end
|
26
|
+
|
27
|
+
def notify(key, message = nil)
|
28
|
+
sql = "NOTIFY #{key}"
|
29
|
+
sql << ", '#{message}'" if message
|
30
|
+
exec(sql)
|
31
|
+
end
|
32
|
+
|
33
|
+
def listen(key)
|
34
|
+
exec("LISTEN #{key}")
|
35
|
+
end
|
36
|
+
|
37
|
+
def unlisten(key)
|
38
|
+
exec("UNLISTEN #{key}")
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/pg_queue/version.rb
CHANGED
data/lib/pg_queue/worker.rb
CHANGED
@@ -1,47 +1,33 @@
|
|
1
1
|
module PgQueue
|
2
2
|
class Worker
|
3
|
-
attr_reader :connection, :interval
|
4
|
-
|
5
|
-
def initialize
|
6
|
-
@connection = new_connection
|
7
|
-
@queue = PgQueue::Queue.new(@connection)
|
8
|
-
@interval = 5
|
9
|
-
end
|
10
|
-
|
11
3
|
def start
|
12
4
|
@running = true
|
13
5
|
|
14
|
-
listen
|
6
|
+
PgQueue.connection.listen(:pg_queue_jobs)
|
15
7
|
while running?
|
16
|
-
job =
|
8
|
+
job = PgQueue.dequeue
|
17
9
|
if job
|
18
10
|
perform(job)
|
19
11
|
next
|
20
12
|
end
|
21
13
|
|
22
|
-
|
23
|
-
connection.wait_for_notify do |event, pid, payload|
|
14
|
+
PgQueue.logger.debug("waiting for jobs")
|
15
|
+
PgQueue.connection.wait_for_notify do |event, pid, payload|
|
24
16
|
if payload == "stop"
|
25
|
-
|
17
|
+
PgQueue.logger.debug("stop notify received")
|
26
18
|
stop
|
27
19
|
else
|
28
|
-
|
20
|
+
PgQueue.logger.debug("let's perform some jobs")
|
29
21
|
end
|
30
22
|
end
|
31
23
|
end
|
32
|
-
|
33
|
-
|
34
|
-
def listen
|
35
|
-
connection.exec("LISTEN pg_queue_jobs")
|
36
|
-
end
|
37
|
-
|
38
|
-
def unlisten
|
39
|
-
connection.exec("UNLISTEN pg_queue_jobs")
|
24
|
+
PgQueue.connection.unlisten(:pg_queue_jobs)
|
40
25
|
end
|
41
26
|
|
42
27
|
def stop
|
43
28
|
@running = false
|
44
|
-
|
29
|
+
PgQueue.connection = nil
|
30
|
+
PgQueue.connection.notify(:pg_queue_jobs, "stop")
|
45
31
|
end
|
46
32
|
|
47
33
|
def running?
|
@@ -50,17 +36,13 @@ module PgQueue
|
|
50
36
|
|
51
37
|
protected
|
52
38
|
|
53
|
-
def new_connection
|
54
|
-
PGconn.open(:dbname => 'pg_queue_test')
|
55
|
-
end
|
56
|
-
|
57
39
|
def perform(job)
|
58
40
|
begin
|
59
|
-
|
41
|
+
PgQueue.logger.debug(job.inspect)
|
60
42
|
job.perform
|
61
43
|
rescue => ex
|
62
|
-
|
63
|
-
|
44
|
+
PgQueue.logger.fatal(ex)
|
45
|
+
PgQueue.logger.fatal(ex.message)
|
64
46
|
end
|
65
47
|
end
|
66
48
|
end
|
data/lib/pg_queue.rb
CHANGED
@@ -1,8 +1,45 @@
|
|
1
1
|
require "pg_queue/version"
|
2
2
|
require "pg"
|
3
|
+
require "logger"
|
4
|
+
require "multi_json"
|
3
5
|
|
4
6
|
module PgQueue
|
5
7
|
autoload :Worker, "pg_queue/worker"
|
6
|
-
autoload :Queue, "pg_queue/queue"
|
7
8
|
autoload :Job, "pg_queue/job"
|
9
|
+
autoload :PgExtensions, "pg_queue/pg_extensions"
|
10
|
+
|
11
|
+
def self.logger
|
12
|
+
@logger ||= Logger.new(STDOUT)
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.logger=(object)
|
16
|
+
@logger = object
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.connection
|
20
|
+
@connection ||= begin
|
21
|
+
conn = PGconn.open(:dbname => 'pg_queue_test')
|
22
|
+
conn.extend(PgQueue::PgExtensions)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.connection=(object)
|
27
|
+
@connection = object
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.enqueue(klass, *args)
|
31
|
+
id = connection.insert(:pg_queue_jobs, { :class_name => klass.name, :args => MultiJson.encode(args) }, "id")
|
32
|
+
logger.debug("enqueued #{id}")
|
33
|
+
connection.notify(:pg_queue_jobs)
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.dequeue
|
37
|
+
result = connection.first("SELECT id, class_name, args FROM pg_queue_jobs LIMIT 1")
|
38
|
+
return nil unless result
|
39
|
+
|
40
|
+
PgQueue::Job.new(result).tap do |job|
|
41
|
+
PgQueue.logger.debug("dequeued #{job.id}")
|
42
|
+
connection.delete(:pg_queue_jobs, :id => job.id)
|
43
|
+
end
|
44
|
+
end
|
8
45
|
end
|
data/pg_queue.gemspec
CHANGED
@@ -0,0 +1,22 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
class MyQueue; end
|
4
|
+
|
5
|
+
describe PgQueue::Job do
|
6
|
+
subject do
|
7
|
+
described_class.new(
|
8
|
+
"id" => 1,
|
9
|
+
"class_name" => "MyQueue",
|
10
|
+
"args" => MultiJson.encode([1, "two"])
|
11
|
+
)
|
12
|
+
end
|
13
|
+
|
14
|
+
its(:id) { should == 1 }
|
15
|
+
its(:klass) { should == MyQueue }
|
16
|
+
its(:args) { should == [1, "two"] }
|
17
|
+
|
18
|
+
it "performs the job" do
|
19
|
+
subject.klass.should_receive(:perform)
|
20
|
+
subject.perform
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe PgQueue::Worker do
|
4
|
+
subject { described_class.new }
|
5
|
+
before do
|
6
|
+
PgQueue.connection = double("connection").as_null_object
|
7
|
+
end
|
8
|
+
|
9
|
+
context "starting" do
|
10
|
+
before do
|
11
|
+
PgQueue.connection.should_receive(:listen).with(:pg_queue_jobs)
|
12
|
+
[true, false].each { |ret| subject.should_receive(:running?).and_return(ret) }
|
13
|
+
PgQueue.connection.should_receive(:unlisten).with(:pg_queue_jobs)
|
14
|
+
end
|
15
|
+
|
16
|
+
context "with jobs pending" do
|
17
|
+
let(:job) { double("job") }
|
18
|
+
|
19
|
+
it "performs the job" do
|
20
|
+
PgQueue.should_receive(:dequeue).and_return(job)
|
21
|
+
job.should_receive(:perform)
|
22
|
+
|
23
|
+
subject.start
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
context "with no jobs pending" do
|
28
|
+
it "waits for notifies" do
|
29
|
+
PgQueue.should_receive(:dequeue).and_return(nil)
|
30
|
+
PgQueue.connection.should_receive(:wait_for_notify).and_yield("pg_queue_jobs", "12345", nil)
|
31
|
+
|
32
|
+
subject.start
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
context "stoping" do
|
38
|
+
let(:connection) { double("connection") }
|
39
|
+
|
40
|
+
before do
|
41
|
+
subject.instance_variable_set(:@running, true)
|
42
|
+
end
|
43
|
+
|
44
|
+
it "notifies worker to stop" do
|
45
|
+
PgQueue.should_receive(:connection=).with(nil)
|
46
|
+
PgQueue.should_receive(:connection).and_return(connection)
|
47
|
+
connection.should_receive(:notify).with(:pg_queue_jobs, "stop")
|
48
|
+
|
49
|
+
subject.should be_running
|
50
|
+
subject.stop
|
51
|
+
subject.should_not be_running
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
class MyLogger; end
|
4
|
+
class MyDbConnection; end
|
5
|
+
class MyQueue; end
|
6
|
+
|
7
|
+
describe PgQueue do
|
8
|
+
context "logging" do
|
9
|
+
its(:logger) { should be_instance_of(Logger) }
|
10
|
+
|
11
|
+
it "defines a new logger" do
|
12
|
+
described_class.logger = MyLogger.new
|
13
|
+
described_class.logger.should be_instance_of(MyLogger)
|
14
|
+
end
|
15
|
+
|
16
|
+
after do
|
17
|
+
described_class.logger = nil
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
context "database connection" do
|
22
|
+
its(:connection) { should be_instance_of(PGconn) }
|
23
|
+
|
24
|
+
context "extensions" do
|
25
|
+
subject { described_class.connection }
|
26
|
+
it { should respond_to(:insert) }
|
27
|
+
it { should respond_to(:notify) }
|
28
|
+
it { should respond_to(:delete) }
|
29
|
+
it { should respond_to(:first) }
|
30
|
+
it { should respond_to(:listen) }
|
31
|
+
it { should respond_to(:unlisten) }
|
32
|
+
end
|
33
|
+
|
34
|
+
it "defines a new connection" do
|
35
|
+
described_class.connection = MyDbConnection.new
|
36
|
+
described_class.connection.should be_instance_of(MyDbConnection)
|
37
|
+
end
|
38
|
+
|
39
|
+
after do
|
40
|
+
described_class.connection = nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
context "enqueuing/dequeuing" do
|
45
|
+
let(:connection) { described_class.connection }
|
46
|
+
let(:result) { double("result") }
|
47
|
+
|
48
|
+
it "enqueues a job" do
|
49
|
+
connection.should_receive(:insert).with(:pg_queue_jobs, instance_of(Hash), "id").and_return("1")
|
50
|
+
connection.should_receive(:notify).with(:pg_queue_jobs)
|
51
|
+
|
52
|
+
described_class.enqueue(MyQueue, 1, "two")
|
53
|
+
end
|
54
|
+
|
55
|
+
it "dequeues a job" do
|
56
|
+
connection.should_receive(:first).with(/LIMIT 1/).and_return(result)
|
57
|
+
result.should_receive(:[]).with("id").and_return("1")
|
58
|
+
result.should_receive(:[]).with("class_name").and_return("MyQueue")
|
59
|
+
result.should_receive(:[]).with("args").and_return("[1, \"two\"]")
|
60
|
+
connection.should_receive(:delete).with(:pg_queue_jobs, :id => "1")
|
61
|
+
|
62
|
+
subject.dequeue.should be_instance_of(PgQueue::Job)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pg_queue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.2
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Rafael Souza
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-01-
|
12
|
+
date: 2012-01-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: pg
|
16
|
-
requirement: &
|
16
|
+
requirement: &70278043238320 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,7 +21,29 @@ dependencies:
|
|
21
21
|
version: 0.12.2
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70278043238320
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: multi_json
|
27
|
+
requirement: &70278043237820 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ~>
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 1.0.4
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *70278043237820
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: rspec
|
38
|
+
requirement: &70278043237360 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ~>
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 2.8.0
|
44
|
+
type: :development
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *70278043237360
|
25
47
|
description: Some experimentations with LISTEN/NOTIFY for background jobs
|
26
48
|
email:
|
27
49
|
- me@rafaelss.com
|
@@ -31,6 +53,7 @@ extensions: []
|
|
31
53
|
extra_rdoc_files: []
|
32
54
|
files:
|
33
55
|
- .gitignore
|
56
|
+
- .rspec
|
34
57
|
- Gemfile
|
35
58
|
- LICENSE
|
36
59
|
- README.md
|
@@ -38,10 +61,14 @@ files:
|
|
38
61
|
- bin/queue_worker
|
39
62
|
- lib/pg_queue.rb
|
40
63
|
- lib/pg_queue/job.rb
|
41
|
-
- lib/pg_queue/
|
64
|
+
- lib/pg_queue/pg_extensions.rb
|
42
65
|
- lib/pg_queue/version.rb
|
43
66
|
- lib/pg_queue/worker.rb
|
44
67
|
- pg_queue.gemspec
|
68
|
+
- spec/pg_queue/job_spec.rb
|
69
|
+
- spec/pg_queue/worker_spec.rb
|
70
|
+
- spec/pg_queue_spec.rb
|
71
|
+
- spec/spec_helper.rb
|
45
72
|
homepage: http://github.com/rafaelss/pg_queue
|
46
73
|
licenses: []
|
47
74
|
post_install_message:
|
@@ -57,13 +84,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
57
84
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
58
85
|
none: false
|
59
86
|
requirements:
|
60
|
-
- - ! '
|
87
|
+
- - ! '>='
|
61
88
|
- !ruby/object:Gem::Version
|
62
|
-
version:
|
89
|
+
version: '0'
|
63
90
|
requirements: []
|
64
91
|
rubyforge_project:
|
65
92
|
rubygems_version: 1.8.10
|
66
93
|
signing_key:
|
67
94
|
specification_version: 3
|
68
95
|
summary: Background jobs using PostgreSQL's LISTEN/NOTIFY
|
69
|
-
test_files:
|
96
|
+
test_files:
|
97
|
+
- spec/pg_queue/job_spec.rb
|
98
|
+
- spec/pg_queue/worker_spec.rb
|
99
|
+
- spec/pg_queue_spec.rb
|
100
|
+
- spec/spec_helper.rb
|
data/lib/pg_queue/queue.rb
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
require "json"
|
2
|
-
|
3
|
-
module PgQueue
|
4
|
-
class Queue
|
5
|
-
attr_reader :connection
|
6
|
-
|
7
|
-
def initialize(connection)
|
8
|
-
@connection = connection
|
9
|
-
end
|
10
|
-
|
11
|
-
def enqueue(klass, *args)
|
12
|
-
sql = "INSERT INTO pg_queue_jobs (klass, args) VALUES ($1, $2) RETURNING id"
|
13
|
-
id = connection.exec(sql, [klass.name, JSON.dump(args)]).getvalue(0, 0)
|
14
|
-
puts "enqueued #{id}"
|
15
|
-
connection.exec("NOTIFY pg_queue_jobs")
|
16
|
-
end
|
17
|
-
|
18
|
-
def dequeue
|
19
|
-
result = connection.exec("SELECT id, klass, args FROM pg_queue_jobs LIMIT 1")
|
20
|
-
return nil unless result.count == 1
|
21
|
-
PgQueue::Job.new(result[0]).tap do |job|
|
22
|
-
connection.exec("DELETE FROM pg_queue_jobs WHERE id = $1", [job.id])
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|