queue_classic 0.1.6 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/notifier.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  class Notifier
2
+
2
3
  def self.deliver(msg)
3
- `say #{msg}`
4
+ `say #{msg}`
4
5
  end
6
+
5
7
  end
data/lib/queue_classic.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  require 'json'
2
2
  require 'pg'
3
+ require 'uri'
3
4
 
4
5
  $: << File.expand_path("lib")
5
6
 
@@ -7,9 +8,10 @@ require 'queue_classic/durable_array'
7
8
  require 'queue_classic/worker'
8
9
  require 'queue_classic/queue'
9
10
  require 'queue_classic/api'
11
+ require 'queue_classic/job'
10
12
 
11
13
  module QC
12
14
  extend Api
13
15
  end
14
16
 
15
- QC::Queue.setup :data_store => QC::DurableArray.new(:database => ENV["DATABASE_URL"])
17
+ QC::Queue.instance.setup :data_store => QC::DurableArray.new(:database => ENV["DATABASE_URL"])
@@ -1,20 +1,24 @@
1
1
  module QC
2
2
  module Api
3
3
 
4
+ def queue
5
+ @queue ||= Queue.instance
6
+ end
7
+
4
8
  def enqueue(job,*params)
5
- Queue.enqueue(job,params)
9
+ queue.enqueue(job,params)
6
10
  end
7
11
 
8
12
  def dequeue
9
- Queue.dequeue
13
+ queue.dequeue
10
14
  end
11
15
 
12
16
  def delete(job)
13
- Queue.delete(job)
17
+ queue.delete(job)
14
18
  end
15
19
 
16
20
  def queue_length
17
- Queue.length
21
+ queue.length
18
22
  end
19
23
 
20
24
  def work(job)
@@ -23,13 +27,10 @@ module QC
23
27
  params = job.params
24
28
 
25
29
  klass.send(method,params)
26
- delete(job)
27
- rescue ArgumentError => e
28
- puts "ArgumentError: #{e.inspect}"
29
30
  end
30
31
 
31
32
  def logging_enabled?
32
- true
33
+ ENV["LOGGING"]
33
34
  end
34
35
 
35
36
  end
@@ -1,35 +1,6 @@
1
- require 'uri'
2
- require 'pg'
3
-
4
1
  module QC
5
- class Job
6
- attr_accessor :id, :details, :locked_at
7
- def initialize(args={})
8
- @id = args["id"]
9
- @details = args["details"]
10
- @locked_at = args["locked_at"]
11
- end
12
-
13
- def klass
14
- Kernel.const_get(details["job"].split(".").first)
15
- end
16
-
17
- def method
18
- details["job"].split(".").last
19
- end
20
-
21
- def params
22
- params = details["params"]
23
- if params.length == 1
24
- return params[0]
25
- else
26
- params
27
- end
28
- end
29
-
30
- end
31
-
32
2
  class DurableArray
3
+
33
4
  def initialize(args={})
34
5
  @db_string = args[:database]
35
6
  @connection = connection
@@ -55,26 +26,17 @@ module QC
55
26
  find_one {"SELECT * FROM jobs WHERE id = #{job.id}"}
56
27
  end
57
28
 
58
- def head
59
- find_one {"SELECT * FROM jobs ORDER BY id ASC LIMIT 1"}
60
- end
61
- alias :first :head
62
-
63
29
  def lock_head
64
30
  job = nil
65
- with_log("start lock transaction") do
66
- @connection.transaction do
67
- if job = find_one {"SELECT * FROM jobs WHERE locked_at IS NULL ORDER BY id ASC LIMIT 1 FOR UPDATE"}
68
- with_log("lock acquired for #{job.inspect}") do
69
- execute("UPDATE jobs SET locked_at = (CURRENT_TIMESTAMP) WHERE id = #{job.id} AND locked_at IS NULL")
70
- end
71
- end
31
+ @connection.transaction do
32
+ if job = find_one {"SELECT * FROM jobs WHERE locked_at IS NULL ORDER BY id ASC LIMIT 1 FOR UPDATE"}
33
+ execute("UPDATE jobs SET locked_at = (CURRENT_TIMESTAMP) WHERE id = #{job.id} AND locked_at IS NULL")
72
34
  end
73
35
  end
74
36
  job
75
37
  end
76
38
 
77
- def b_head
39
+ def first
78
40
  if job = lock_head
79
41
  job
80
42
  else
@@ -0,0 +1,29 @@
1
+ module QC
2
+ class Job
3
+ attr_accessor :id, :details, :locked_at
4
+
5
+ def initialize(args={})
6
+ @id = args["id"]
7
+ @details = args["details"]
8
+ @locked_at = args["locked_at"]
9
+ end
10
+
11
+ def klass
12
+ Kernel.const_get(details["job"].split(".").first)
13
+ end
14
+
15
+ def method
16
+ details["job"].split(".").last
17
+ end
18
+
19
+ def params
20
+ params = details["params"]
21
+ if params.length == 1
22
+ return params[0]
23
+ else
24
+ params
25
+ end
26
+ end
27
+
28
+ end
29
+ end
@@ -1,25 +1,26 @@
1
+ require 'singleton'
2
+
1
3
  module QC
2
4
  class Queue
3
- def self.setup(args={})
4
- @@data = args[:data_store] || []
5
- self
5
+ include Singleton
6
+ def setup(args={})
7
+ @data = args[:data_store]
6
8
  end
7
9
 
8
- def self.enqueue(job,params)
9
- @@data << {"job" => job, "params" => params}
10
+ def enqueue(job,params)
11
+ @data << {"job" => job, "params" => params}
10
12
  end
11
13
 
12
- def self.dequeue
13
- @@data.b_head
14
+ def dequeue
15
+ @data.first
14
16
  end
15
17
 
16
- def self.delete(job)
17
- @@data.delete(job)
18
+ def delete(job)
19
+ @data.delete(job)
18
20
  end
19
21
 
20
- def self.length
21
- @@data.count
22
+ def length
23
+ @data.count
22
24
  end
23
-
24
25
  end
25
26
  end
@@ -1,24 +1,29 @@
1
1
  module QC
2
2
  class Worker
3
3
 
4
- def initialize
5
- @worker_id = rand(1000)
6
- end
7
-
8
4
  def start
9
- puts "#{@worker_id} ready for work"
10
5
  loop { work }
11
6
  end
12
7
 
13
8
  def work
14
- job = QC.dequeue
15
- # if we are here, dequeue has unblocked
16
- # and we may have a job.
17
- if job
18
- puts "#{@worker_id} working job"
19
- QC.work(job)
9
+ if job = QC.dequeue #blocks until we have a job
10
+ begin
11
+ QC.work(job)
12
+ rescue Object => e
13
+ handle_failure(job,e)
14
+ ensure
15
+ QC.delete(job)
16
+ end
20
17
  end
18
+ end
21
19
 
20
+ #override this method to do whatever you want
21
+ def handle_failure(job,e)
22
+ puts "!"
23
+ puts "! \t FAIL"
24
+ puts "! \t \t #{job.inspect}"
25
+ puts "! \t \t #{e.inspect}"
26
+ puts "!"
22
27
  end
23
28
 
24
29
  end
data/readme.markdown CHANGED
@@ -1,7 +1,9 @@
1
1
  # Queue Classic
2
- __Alpha 0.1.6__
2
+ __Beta 0.2.0__
3
3
 
4
- _Queue Classic 0.1.6 is not ready for production. However, it is under active development and I expect a beta release within the following months._
4
+ __Queue Classic 0.2.0 is in Beta.__ I have been using this library with 30-50 Heroku workers and have had great results. However, your mileage may vary.
5
+
6
+ I am using this in production applications and plan to maintain and support this library for a long time.
5
7
 
6
8
  Queue Classic is an alternative queueing library for Ruby apps (Rails, Sinatra, Etc...) Queue Classic features __asynchronous__ job polling, database maintained locks and
7
9
  no ridiculous dependencies. As a matter of fact, Queue Classic only requires the __pg__ and __json__.
@@ -14,6 +16,12 @@ no ridiculous dependencies. As a matter of fact, Queue Classic only requires the
14
16
  3. QC.enqueue "Class.method", :arg1 => val1
15
17
  4. rake qc:work
16
18
 
19
+ ### Dependencies
20
+
21
+ Postgres version 9
22
+
23
+ Ruby (gems: pg, json)
24
+
17
25
  ### Gem
18
26
 
19
27
  gem install queue_classic
@@ -31,6 +39,7 @@ Once your Database is set, you will need to add a jobs table. If you are using r
31
39
  create_table :jobs do |t|
32
40
  t.text :details
33
41
  t.timestamp :locked_at
42
+ t.index :id
34
43
  end
35
44
  end
36
45
 
@@ -40,6 +49,9 @@ Once your Database is set, you will need to add a jobs table. If you are using r
40
49
  end
41
50
  After running this migration, your database should be ready to go. As a sanity check, enqueue a job and then issue a SELECT in the postgres console.
42
51
 
52
+ Be sure and add the index to the id column. This will help out the worker if the queue should ever reach an obscene length. It made a huge difference
53
+ when running the benchmark.
54
+
43
55
  __script/console__
44
56
  QC.enqueue "Class.method"
45
57
  __Terminal__
@@ -87,13 +99,40 @@ Finally, the strongest feature of Queue Classic is it's ability to block on on d
87
99
  of the wonderul PUB/SUB featuers built in to Postgres. Basically there is a channel in which the workers LISTEN. When a new job is added to the queue, the queue sends NOTIFY
88
100
  messages on the channel. Once a NOTIFY is sent, each worker races to acquire a lock on a job. A job is awareded to the victor while the rest go back to wait for another job.
89
101
 
102
+ ### The Worker
103
+
104
+ The worker calls dequeue and then calls the enqueued method with the supplied arguments. Once the method terminates, the job is deleted from the queue. In the case that your method
105
+ does not terminate, or the worker unexpectingly dies, Queue Classic will do following: It will ensure that the job is deleted from the queue and it will call a method (which is certainly over
106
+ rideable) that handles the failure. You can do whatever you want in the handle_failure method, log to Get Exceptional, enqueue the job again, log to stderror.
107
+
108
+ class MyWorker < QC::Worker
109
+ def handle_failure(job,exception)
110
+ puts job.inspect
111
+ puts exception.inspect
112
+ end
113
+ end
114
+
115
+ worker = MyWorker.new
116
+ worker.start
117
+
118
+ ## Performance
119
+ I am pleased at the performance of Queue Classic. It ran 3x faster than the some of the most popular Relational Database backed queues. (I have yet to benchmark redis backed queues)
120
+
121
+ ruby benchmark.rb
122
+ user system total real
123
+ 0.950000 0.620000 1.570000 ( 9.479941)
124
+
125
+ Hardware: Mac Book Pro 2.8 GHz Intel Core i7. SSD. 4 GB memory.
126
+
127
+ Software: Ruby 1.9.2-p0, PostgreSQL 9.0.2
128
+
90
129
  ## FAQ
91
130
 
92
131
  How is this different than DJ?
93
132
  > TL;DR = Store job as JSON (better introspection), Queue manages the time for locking jobs (workers can be out of sync.), No magic (less code), Small footprint (ORM Free).
94
133
 
95
134
  > __Introspection__ I want the data in the queue to be as simple as possible. Since we only store the Class, Method and Args, introspection into the queue is
96
- quite simpler.
135
+ quite simple.
97
136
 
98
137
  > __Locking__ You might have noticed that DJ's worker calls Time.now(). In a cloud environment, this could allow for workers to be confused about
99
138
  the status of a job. Classic Queue locks a job using Postgres' TIMESTAMP function.
@@ -111,8 +150,7 @@ creating jobs for the emailing of newsletters; put a emailed_at column on your n
111
150
  and then right before the job quits, touch the emailed_at column.
112
151
 
113
152
  Can I use this library with 50 Heroku Workers?
114
- > Maybe. I haven't tested 50 workers yet. But it is definitely a goal for Queue Classic. I am not sure when,
115
- but you can count on this library being able to handle all Heroku can throw at it.
153
+ > Yes.
116
154
 
117
155
  Why does this project seem incomplete? Will you make it production ready?
118
156
  > I started this project on 1/24/2011. Check back soon! Also, feel free to contact me to find out how passionate I am about queueing.
@@ -23,6 +23,7 @@ module DatabaseHelpers
23
23
  "locked_at timestamp without time zone" +
24
24
  ");"
25
25
  )
26
+ jobs_db.exec("CREATE INDEX jobs_id_idx ON jobs (id)")
26
27
  end
27
28
 
28
29
  def drop_table
@@ -3,12 +3,12 @@ require File.expand_path("../helper.rb", __FILE__)
3
3
  class DurableArrayTest < MiniTest::Unit::TestCase
4
4
  include DatabaseHelpers
5
5
 
6
- def test_head_decodes_json
6
+ def test_first_decodes_json
7
7
  clean_database
8
8
  array = QC::DurableArray.new(:database => "queue_classic_test")
9
9
 
10
10
  array << {"test" => "ok"}
11
- assert_equal({"test" => "ok"}, array.head.details)
11
+ assert_equal({"test" => "ok"}, array.first.details)
12
12
  end
13
13
 
14
14
  def test_count_returns_number_of_rows
@@ -21,22 +21,22 @@ class DurableArrayTest < MiniTest::Unit::TestCase
21
21
  assert_equal 2, array.count
22
22
  end
23
23
 
24
- def test_head_returns_first_job
24
+ def test_first_returns_fsrst_job
25
25
  clean_database
26
26
  array = QC::DurableArray.new(:database => "queue_classic_test")
27
27
 
28
28
  job = {"job" => "one"}
29
29
  array << job
30
- assert_equal job, array.head.details
30
+ assert_equal job, array.first.details
31
31
  end
32
32
 
33
- def test_head_returns_first_job_when_many_are_in_array
33
+ def test_first_returns_first_job_when_many_are_in_array
34
34
  clean_database
35
35
  array = QC::DurableArray.new(:database => "queue_classic_test")
36
36
 
37
37
  array << {"job" => "one"}
38
38
  array << {"job" => "two"}
39
- assert_equal({"job" => "one"}, array.head.details)
39
+ assert_equal({"job" => "one"}, array.first.details)
40
40
  end
41
41
 
42
42
  def test_delete_removes_job_from_array
@@ -44,9 +44,11 @@ class DurableArrayTest < MiniTest::Unit::TestCase
44
44
  array = QC::DurableArray.new(:database => "queue_classic_test")
45
45
 
46
46
  array << {"job" => "one"}
47
- assert_equal( {"job" => "one"}, array.head.details)
48
- array.delete(array.head)
49
- assert_nil array.head
47
+ job = array.first
48
+
49
+ assert_equal( {"job" => "one"}, job.details)
50
+ array.delete(job)
51
+ assert_nil array.first
50
52
  end
51
53
 
52
54
  def test_delete_returns_job_after_delete
@@ -54,10 +56,12 @@ class DurableArrayTest < MiniTest::Unit::TestCase
54
56
  array = QC::DurableArray.new(:database => "queue_classic_test")
55
57
 
56
58
  array << {"job" => "one"}
57
- assert_equal({"job" => "one"}, array.head.details)
59
+ job = array.first
60
+
61
+ assert_equal({"job" => "one"}, job.details)
58
62
 
59
- res = array.delete(array.head)
60
- assert_nil(array.head)
63
+ res = array.delete(job)
64
+ assert_nil(array.first)
61
65
  assert_equal({"job" => "one"}, res.details)
62
66
  end
63
67
 
data/test/helper.rb CHANGED
@@ -1,8 +1,18 @@
1
1
  $: << File.expand_path("lib")
2
2
  $: << File.expand_path("test")
3
3
 
4
+ ENV["DATABASE_URL"] = 'queue_classic_test'
5
+
4
6
  require 'queue_classic'
5
7
  require 'database_helpers'
6
8
 
7
9
  require 'minitest/unit'
8
10
  MiniTest::Unit.autorun
11
+
12
+ def set_data_store(store=nil)
13
+ QC::Queue.instance.setup(
14
+ :data_store => (
15
+ store || QC::DurableArray.new(:database => ENV["DATABASE_URL"])
16
+ )
17
+ )
18
+ end
data/test/queue_test.rb CHANGED
@@ -2,4 +2,21 @@ require File.expand_path("../helper.rb", __FILE__)
2
2
 
3
3
  class QueueTest < MiniTest::Unit::TestCase
4
4
  include DatabaseHelpers
5
+
6
+ def test_queue_is_singleton
7
+ assert_equal QC::Queue, QC::Queue.instance.class
8
+ end
9
+
10
+ def test_queue_setup
11
+ QC::Queue.instance.setup :data_store => []
12
+ assert_equal [], QC::Queue.instance.instance_variable_get(:@data)
13
+ end
14
+
15
+ def test_queue_length
16
+ QC::Queue.instance.setup :data_store => []
17
+ QC::Queue.instance.enqueue "job","params"
18
+
19
+ assert_equal 1, QC::Queue.instance.length
20
+ end
21
+
5
22
  end
data/test/worker_test.rb CHANGED
@@ -5,18 +5,46 @@ class TestNotifier
5
5
  end
6
6
  end
7
7
 
8
+ # This not only allows me to test what happens
9
+ # when a failure occures but it also demonstrates
10
+ # how to override the worker to handle failures the way
11
+ # you want.
12
+ class TestWorker < QC::Worker
13
+ attr_accessor :failed_count
14
+ def initialize
15
+ @failed_count = 0
16
+ super
17
+ end
18
+ def handle_failure(job,e)
19
+ @failed_count += 1
20
+ end
21
+ end
22
+
8
23
  class WorkerTest < MiniTest::Unit::TestCase
9
24
  include DatabaseHelpers
10
25
 
11
26
  def test_working_a_job
27
+ set_data_store
12
28
  clean_database
13
29
 
14
30
  QC.enqueue "TestNotifier.deliver", {}
15
- worker = QC::Worker.new
31
+ worker = TestWorker.new
16
32
 
17
33
  assert_equal(1, QC.queue_length)
18
34
  worker.work
19
35
  assert_equal(0, QC.queue_length)
36
+ assert_equal(0, worker.failed_count)
37
+ end
38
+
39
+ def test_rescue_failed_jobs
40
+ set_data_store
41
+ clean_database
42
+
43
+ QC.enqueue "TestNotifier.no_method", {}
44
+ worker = TestWorker.new
45
+
46
+ worker.work
47
+ assert_equal 1, worker.failed_count
20
48
  end
21
49
 
22
50
  end
metadata CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
4
4
  prerelease: false
5
5
  segments:
6
6
  - 0
7
- - 1
8
- - 6
9
- version: 0.1.6
7
+ - 2
8
+ - 0
9
+ version: 0.2.0
10
10
  platform: ruby
11
11
  authors:
12
12
  - Ryan Smith
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2011-01-28 00:00:00 -08:00
17
+ date: 2011-02-16 00:00:00 -08:00
18
18
  default_executable:
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
@@ -58,6 +58,7 @@ files:
58
58
  - lib/notifier.rb
59
59
  - lib/queue_classic/api.rb
60
60
  - lib/queue_classic/durable_array.rb
61
+ - lib/queue_classic/job.rb
61
62
  - lib/queue_classic/queue.rb
62
63
  - lib/queue_classic/tasks.rb
63
64
  - lib/queue_classic/worker.rb