queue_classic 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,15 @@
1
+ require 'json'
2
+ require 'pg'
3
+
4
+ $: << File.expand_path("lib")
5
+
6
+ require 'queue_classic/durable_array'
7
+ require 'queue_classic/worker'
8
+ require 'queue_classic/queue'
9
+ require 'queue_classic/api'
10
+
11
+ QC::Queue.setup :data_store => QC::DurableArray.new(:database => ENV["DATABASE_URL"])
12
+
13
+ module QC
14
+ extend Api
15
+ end
@@ -0,0 +1,32 @@
1
+ module QC
2
+ module Api
3
+
4
+ def enqueue(job,*params)
5
+ Queue.enqueue(job,params)
6
+ end
7
+
8
+ def dequeue
9
+ Queue.dequeue
10
+ end
11
+
12
+ def delete(job)
13
+ Queue.delete(job)
14
+ end
15
+
16
+ def queue_length
17
+ Queue.length
18
+ end
19
+
20
+ def work(job)
21
+ klass = job.klass
22
+ method = job.method
23
+ params = job.params
24
+
25
+ klass.send(method,params)
26
+ delete(job)
27
+ rescue ArgumentError => e
28
+ puts "ArgumentError: #{e.inspect}"
29
+ end
30
+
31
+ end
32
+ end
@@ -0,0 +1,124 @@
1
+ require 'uri'
2
+ require 'pg'
3
+
4
+ module QC
5
+ class Job
6
+ attr_accessor :id, :details, :locked_at
7
+ def initialize(args={})
8
+ @id = args["id"]
9
+ @details = args["details"]
10
+ @locked_at = args["locked_at"]
11
+ end
12
+
13
+ def klass
14
+ Kernel.const_get(details["job"].split(".").first)
15
+ end
16
+
17
+ def method
18
+ details["job"].split(".").last
19
+ end
20
+
21
+ def params
22
+ params = details["params"]
23
+ if params.length == 1
24
+ return params[0]
25
+ else
26
+ params
27
+ end
28
+ end
29
+
30
+ end
31
+
32
+ class DurableArray
33
+ def initialize(args={})
34
+ @db_string = args[:database]
35
+ @connection = connection
36
+ execute("SET client_min_messages TO 'warning'")
37
+ execute("LISTEN jobs")
38
+ end
39
+
40
+ def <<(details)
41
+ execute(
42
+ "INSERT INTO jobs" +
43
+ "(details)" +
44
+ "VALUES ('#{details.to_json}')"
45
+ )
46
+ execute("NOTIFY jobs, 'new-job'")
47
+ end
48
+
49
+ def count
50
+ execute("SELECT COUNT(*) from jobs")[0]["count"].to_i
51
+ end
52
+
53
+ def delete(job)
54
+ execute("DELETE FROM jobs WHERE id = #{job.id}")
55
+ job
56
+ end
57
+
58
+ def find(job)
59
+ find_one {"SELECT * FROM jobs WHERE id = #{job.id}"}
60
+ end
61
+
62
+ def head
63
+ find_one {"SELECT * FROM jobs ORDER BY id ASC LIMIT 1"}
64
+ end
65
+ alias :first :head
66
+
67
+ def lock_head
68
+ job = nil
69
+ @connection.transaction do
70
+ job = find_one {"SELECT * FROM jobs WHERE locked_at IS NULL ORDER BY id ASC LIMIT 1 FOR UPDATE"}
71
+ return nil unless job
72
+ locked = execute("UPDATE jobs SET locked_at = (CURRENT_TIMESTAMP) WHERE id = #{job.id} AND locked_at IS NULL")
73
+ end
74
+ job
75
+ end
76
+
77
+ def b_head
78
+ if job = lock_head
79
+ job
80
+ else
81
+ @connection.wait_for_notify {|e,p,msg| job = lock_head if msg == "new-job" }
82
+ job
83
+ end
84
+ end
85
+
86
+ def each
87
+ execute("SELECT * FROM jobs ORDER BY id ASC").each do |r|
88
+ yield(JSON.parse(r["details"]))
89
+ end
90
+ end
91
+
92
+ def execute(sql)
93
+ @connection.async_exec(sql)
94
+ end
95
+
96
+ def find_one
97
+ res = execute(yield)
98
+ if res.cmd_tuples > 0
99
+ res.map do |r|
100
+ Job.new(
101
+ "id" => r["id"],
102
+ "details" => JSON.parse( r["details"]),
103
+ "locked_at" => r["locked_at"]
104
+ )
105
+ end.pop
106
+ end
107
+ end
108
+
109
+ def connection
110
+ db_params = URI.parse(@db_string)
111
+ if db_params.scheme == "postgres"
112
+ PGconn.connect(
113
+ :dbname => db_params.path.gsub("/",""),
114
+ :user => db_params.user,
115
+ :password => db_params.password,
116
+ :host => db_params.host
117
+ )
118
+ else
119
+ PGconn.connect(:dbname => @db_string)
120
+ end
121
+ end
122
+
123
+ end
124
+ end
@@ -0,0 +1,25 @@
1
+ module QC
2
+ class Queue
3
+ def self.setup(args={})
4
+ @@data = args[:data_store] || []
5
+ self
6
+ end
7
+
8
+ def self.enqueue(job,params)
9
+ @@data << {"job" => job, "params" => params}
10
+ end
11
+
12
+ def self.dequeue
13
+ @@data.b_head
14
+ end
15
+
16
+ def self.delete(job)
17
+ @@data.delete(job)
18
+ end
19
+
20
+ def self.length
21
+ @@data.count
22
+ end
23
+
24
+ end
25
+ end
@@ -0,0 +1,13 @@
1
+ namespace :jobs do
2
+ task :work => :environment do
3
+ QC::Worker.new.start
4
+ end
5
+ end
6
+ namespace :qc do
7
+ task :work do
8
+ QC::Worker.new.start
9
+ end
10
+ task :jobs do
11
+ QC.queue_length
12
+ end
13
+ end
@@ -0,0 +1,25 @@
1
+ module QC
2
+ class Worker
3
+
4
+ def initialize
5
+ @worker_id = rand(1000)
6
+ end
7
+
8
+ def start
9
+ puts "#{@worker_id} ready for work"
10
+ loop { work }
11
+ end
12
+
13
+ def work
14
+ job = QC.dequeue
15
+ # if we are here, dequeue has unblocked
16
+ # and we may have a job.
17
+ if job
18
+ puts "#{@worker_id} working job"
19
+ QC.work(job)
20
+ end
21
+
22
+ end
23
+
24
+ end
25
+ end
data/readme.markdown ADDED
@@ -0,0 +1,118 @@
1
+ # Queue Classic
2
+ __Alpha 0.1.5__
3
+
4
+ _Queue Classic 0.1.5 is not ready for production. However, it is under active development and I expect a beta release within the following months._
5
+
6
+ Queue Classic is an alternative queueing library for Ruby apps (Rails, Sinatra, Etc...) Queue Classic features __asynchronous__ job polling, database maintained locks and
7
+ no ridiculous dependencies. As a matter of fact, Queue Classic only requires the __pg__ and __json__.
8
+
9
+ ## Installation
10
+
11
+ ### TL;DR
12
+ 1. gem install queue_classic
13
+ 2. add jobs table to your database
14
+ 3. QC.enqueue "Class.method", :arg1 => val1
15
+ 4. rake qc:work
16
+
17
+ ### Gem
18
+
19
+ gem install queue_classic
20
+
21
+ ### Database
22
+
23
+ Queue Classic needs a database, so make sure that DATABASE_URL points to your database. If you are unsure about whether this var is set, run:
24
+ echo $DATABASE_URL
25
+ in your shell. If you are using Heroku, this var is set and pointing to your primary database.
26
+
27
+ Once your Database is set, you will need to add a jobs table. If you are using rails, add a migration with the following tables:
28
+
29
+ class CreateJobsTable < ActiveRecord::Migration
30
+ def self.up
31
+ create_table :jobs do |t|
32
+ t.text :details
33
+ t.timestamp :locked_at
34
+ end
35
+ end
36
+
37
+ def self.down
38
+ drop_table :jobs
39
+ end
40
+ end
41
+ After running this migration, your database should be ready to go. As a sanity check, enqueue a job and then issue a SELECT in the postgres console.
42
+
43
+ __script/console__
44
+ QC.enqueue "Class.method"
45
+ __Terminal__
46
+ psql you_database_name
47
+ select * from jobs;
48
+ You should see the job "Class.method"
49
+
50
+ ### Rakefile
51
+
52
+ As a convenience, I added a rake task that responds to `rake jobs:work` There are also rake tasks in the `qc` name space.
53
+ To get access to these tasks, Add `require 'queue_classic/tasks'` to your Rakefile.
54
+
55
+ ## Fundamentals
56
+
57
+ ### Enqueue
58
+
59
+ To place a job onto the queue, you should specify a class and a class method. The syntax should be:
60
+
61
+ QC.enqueue('Class.method', :arg1 => 'value1', :arg2 => 'value2')
62
+
63
+ The job gets stored in the jobs table with a details field set to: `{ job: Class.method, params: {arg1: value1, arg2: value2}}` (as JSON)
64
+ Class can be any class and method can be anything that Class will respond to. For example:
65
+
66
+ class Invoice < ActiveRecord::Base
67
+ def self.process(invoice_id)
68
+ invoice = find(invoice_id)
69
+ invoice.process!
70
+ end
71
+
72
+ def self.process_all
73
+ Invoice.all do |invoice|
74
+ QC.enqueue "Invoice.process", invoice.id
75
+ end
76
+ end
77
+ end
78
+
79
+
80
+ ### Dequeue
81
+
82
+ Traditionally, a queue's dequeue operation will remove the item from the queue. However, Queue Classic will not delete the item from the queue, it will lock it
83
+ and then the worker will delete the job once it has finished working it. Queue Classic's greatest strength is it's ability to safely lock jobs. Unlike other
84
+ database backed queing libraries, Classic Queue uses the database time to lock. This allows you to be more relaxed about the time synchronization of your worker machines.
85
+
86
+ Finally, the strongest feature of Queue Classic is it's ability to block on on dequeue. This design removes the need to __ Sleep & SELECT. __ Queue Classic takes advantage
87
+ of the wonderul PUB/SUB featuers built in to Postgres. Basically there is a channel in which the workers LISTEN. When a new job is added to the queue, the queue sends NOTIFY
88
+ messages on the channel. Once a NOTIFY is sent, each worker races to acquire a lock on a job. A job is awareded to the victor while the rest go back to wait for another job.
89
+
90
+ ## FAQ
91
+
92
+ How is this different than DJ?
93
+ > TL;DR = Store job as JSON (better introspection), Queue manages the time for locking jobs (workers can be out of sync.), No magic (less code), Small footprint (ORM Free).
94
+
95
+ > __Introspection__ I want the data in the queue to be as simple as possible. Since we only store the Class, Method and Args, introspection into the queue is
96
+ quite simpler.
97
+
98
+ > __Locking__ You might have noticed that DJ's worker calls Time.now(). In a cloud environment, this could allow for workers to be confused about
99
+ the status of a job. Classic Queue locks a job using Postgres' TIMESTAMP function.
100
+
101
+ > __Magic__ I find disdain for methods on my objects that have nothing to do with the purpose of the object. Methods like "should" and "delay"
102
+ are quite distasteful and obscure what is actually going on. If you use TestUnit for this reason, you might like Queue Classic. Anyway, I think
103
+ the fundamental concept of a message queue is not that difficult to grasp; therefore, I have taken the time to make Queue Classic as transparent as possilbe.
104
+
105
+ > __Footprint__ You don't need ActiveRecord or any other ORM to find the head or add to the tail. Take a look at the DurableArray class to see the SQL Classic Queue employees.
106
+
107
+ Why doesn't your queue retry failed jobs?
108
+ > I believe the Class method should handle any sort of exception. Also, I think
109
+ that the model you are working on should know about it's state. For instance, if you are
110
+ creating jobs for the emailing of newsletters; put a emailed_at column on your newsletter model
111
+ and then right before the job quits, touch the emailed_at column.
112
+
113
+ Can I use this library with 50 Heroku Workers?
114
+ > Maybe. I haven't tested 50 workers yet. But it is definitely a goal for Queue Classic. I am not sure when,
115
+ but you can count on this library being able to handle all Heroku can throw at it.
116
+
117
+ Why does this project seem incomplete? Will you make it production ready?
118
+ > I started this project on 1/24/2011. Check back soon! Also, feel free to contact me to find out how passionate I am about queueing.
data/test/api_test.rb ADDED
@@ -0,0 +1,14 @@
1
+ require File.expand_path("../helper.rb", __FILE__)
2
+
3
+ class ApiTest < MiniTest::Unit::TestCase
4
+ include DatabaseHelpers
5
+
6
+ def test_enqueue_takes_a_job
7
+ clean_database
8
+
9
+ assert_equal 0, QC.queue_length
10
+ res = QC.enqueue "Notifier.send", {}
11
+ assert_equal 1, QC.queue_length
12
+ end
13
+ end
14
+
@@ -0,0 +1,48 @@
1
+ module DatabaseHelpers
2
+
3
+ def clean_database
4
+ drop_table
5
+ create_table
6
+ disconnect
7
+ end
8
+
9
+ def create_database
10
+ postgres.exec "CREATE DATABASE #{ENV['DATABASE_URL']}"
11
+ end
12
+
13
+ def drop_database
14
+ postgres.exec "DROP DATABASE IF EXISTS #{ENV['DATABASE_URL']}"
15
+ end
16
+
17
+ def create_table
18
+ jobs_db.exec(
19
+ "CREATE TABLE jobs" +
20
+ "(" +
21
+ "id SERIAL," +
22
+ "details text," +
23
+ "locked_at timestamp without time zone" +
24
+ ");"
25
+ )
26
+ end
27
+
28
+ def drop_table
29
+ jobs_db.exec("DROP TABLE IF EXISTS jobs")
30
+ end
31
+
32
+ def disconnect
33
+ jobs_db.finish
34
+ postgres.finish
35
+ end
36
+
37
+ def jobs_db
38
+ @jobs_db ||= PGconn.connect(:dbname => ENV["DATABASE_URL"])
39
+ @jobs_db.exec("SET client_min_messages TO 'warning'")
40
+ @jobs_db
41
+ end
42
+
43
+ def postgres
44
+ @postgres ||= PGconn.connect(:dbname => 'postgres')
45
+ @postgres.exec("SET client_min_messages TO 'warning'")
46
+ @postgres
47
+ end
48
+ end
@@ -0,0 +1,90 @@
1
+ require File.expand_path("../helper.rb", __FILE__)
2
+
3
+ class DurableArrayTest < MiniTest::Unit::TestCase
4
+ include DatabaseHelpers
5
+
6
+ def test_head_decodes_json
7
+ clean_database
8
+ array = QC::DurableArray.new(:database => "queue_classic_test")
9
+
10
+ array << {"test" => "ok"}
11
+ assert_equal({"test" => "ok"}, array.head.details)
12
+ end
13
+
14
+ def test_count_returns_number_of_rows
15
+ clean_database
16
+ array = QC::DurableArray.new(:database => "queue_classic_test")
17
+
18
+ array << {"test" => "ok"}
19
+ assert_equal 1, array.count
20
+ array << {"test" => "ok"}
21
+ assert_equal 2, array.count
22
+ end
23
+
24
+ def test_head_returns_first_job
25
+ clean_database
26
+ array = QC::DurableArray.new(:database => "queue_classic_test")
27
+
28
+ job = {"job" => "one"}
29
+ array << job
30
+ assert_equal job, array.head.details
31
+ end
32
+
33
+ def test_head_returns_first_job_when_many_are_in_array
34
+ clean_database
35
+ array = QC::DurableArray.new(:database => "queue_classic_test")
36
+
37
+ array << {"job" => "one"}
38
+ array << {"job" => "two"}
39
+ assert_equal({"job" => "one"}, array.head.details)
40
+ end
41
+
42
+ def test_delete_removes_job_from_array
43
+ clean_database
44
+ array = QC::DurableArray.new(:database => "queue_classic_test")
45
+
46
+ array << {"job" => "one"}
47
+ assert_equal( {"job" => "one"}, array.head.details)
48
+ array.delete(array.head)
49
+ assert_nil array.head
50
+ end
51
+
52
+ def test_delete_returns_job_after_delete
53
+ clean_database
54
+ array = QC::DurableArray.new(:database => "queue_classic_test")
55
+
56
+ array << {"job" => "one"}
57
+ assert_equal({"job" => "one"}, array.head.details)
58
+
59
+ res = array.delete(array.head)
60
+ assert_nil(array.head)
61
+ assert_equal({"job" => "one"}, res.details)
62
+ end
63
+
64
+ def test_each_yields_the_details_for_each_job
65
+ clean_database
66
+ array = QC::DurableArray.new(:database => "queue_classic_test")
67
+
68
+ array << {"job" => "one"}
69
+ array << {"job" => "two"}
70
+ results = []
71
+ array.each {|v| results << v}
72
+ assert_equal([{"job" => "one"},{"job" => "two"}], results)
73
+ end
74
+
75
+ def test_connection_builds_db_connection_for_uri
76
+ array = QC::DurableArray.new(:database => "postgres://ryandotsmith:@localhost/queue_classic_test")
77
+ assert_equal "ryandotsmith", array.connection.user
78
+ assert_equal "localhost", array.connection.host
79
+ assert_equal "queue_classic_test", array.connection.db
80
+ end
81
+
82
+ def test_connection_builds_db_connection_for_database
83
+ array = QC::DurableArray.new(:database => "queue_classic_test")
84
+
85
+ # FIXME not everyone will have a postgres user named: ryandotsmith
86
+ assert_equal "ryandotsmith", array.connection.user
87
+ assert_equal "queue_classic_test", array.connection.db
88
+ end
89
+
90
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,8 @@
1
+ $: << File.expand_path("lib")
2
+ $: << File.expand_path("test")
3
+
4
+ require 'queue_classic'
5
+ require 'database_helpers'
6
+
7
+ require 'minitest/unit'
8
+ MiniTest::Unit.autorun
@@ -0,0 +1,5 @@
1
+ require File.expand_path("../helper.rb", __FILE__)
2
+
3
+ class QueueTest < MiniTest::Unit::TestCase
4
+ include DatabaseHelpers
5
+ end
@@ -0,0 +1,22 @@
1
+ require File.expand_path("../helper.rb", __FILE__)
2
+
3
+ class TestNotifier
4
+ def self.deliver(args={})
5
+ end
6
+ end
7
+
8
+ class WorkerTest < MiniTest::Unit::TestCase
9
+ include DatabaseHelpers
10
+
11
+ def test_working_a_job
12
+ clean_database
13
+
14
+ QC.enqueue "TestNotifier.deliver", {}
15
+ worker = QC::Worker.new
16
+
17
+ assert_equal(1, QC.queue_length)
18
+ worker.work
19
+ assert_equal(0, QC.queue_length)
20
+ end
21
+
22
+ end
metadata ADDED
@@ -0,0 +1,104 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: queue_classic
3
+ version: !ruby/object:Gem::Version
4
+ prerelease: false
5
+ segments:
6
+ - 0
7
+ - 1
8
+ - 5
9
+ version: 0.1.5
10
+ platform: ruby
11
+ authors:
12
+ - Ryan Smith
13
+ autorequire:
14
+ bindir: bin
15
+ cert_chain: []
16
+
17
+ date: 2011-01-28 00:00:00 -08:00
18
+ default_executable:
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
21
+ name: pg
22
+ prerelease: false
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ none: false
25
+ requirements:
26
+ - - ">="
27
+ - !ruby/object:Gem::Version
28
+ segments:
29
+ - 0
30
+ version: "0"
31
+ type: :runtime
32
+ version_requirements: *id001
33
+ - !ruby/object:Gem::Dependency
34
+ name: json
35
+ prerelease: false
36
+ requirement: &id002 !ruby/object:Gem::Requirement
37
+ none: false
38
+ requirements:
39
+ - - ">="
40
+ - !ruby/object:Gem::Version
41
+ segments:
42
+ - 0
43
+ version: "0"
44
+ type: :runtime
45
+ version_requirements: *id002
46
+ description: Queue Classic is an alternative queueing library for Ruby apps (Rails, Sinatra, Etc...) Queue Classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, Queue Classic only requires the pg and json.
47
+ email: ryan@heroku.com
48
+ executables: []
49
+
50
+ extensions: []
51
+
52
+ extra_rdoc_files: []
53
+
54
+ files:
55
+ - readme.markdown
56
+ - lib/queue_classic/api.rb
57
+ - lib/queue_classic/durable_array.rb
58
+ - lib/queue_classic/queue.rb
59
+ - lib/queue_classic/tasks.rb
60
+ - lib/queue_classic/worker.rb
61
+ - lib/queue_classic.rb
62
+ - test/api_test.rb
63
+ - test/database_helpers.rb
64
+ - test/durable_array_test.rb
65
+ - test/helper.rb
66
+ - test/queue_test.rb
67
+ - test/worker_test.rb
68
+ has_rdoc: true
69
+ homepage: http://github.com/ryandotsmith/Queue-Classic
70
+ licenses: []
71
+
72
+ post_install_message:
73
+ rdoc_options: []
74
+
75
+ require_paths:
76
+ - lib
77
+ required_ruby_version: !ruby/object:Gem::Requirement
78
+ none: false
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ segments:
83
+ - 0
84
+ version: "0"
85
+ required_rubygems_version: !ruby/object:Gem::Requirement
86
+ none: false
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ segments:
91
+ - 0
92
+ version: "0"
93
+ requirements: []
94
+
95
+ rubyforge_project:
96
+ rubygems_version: 1.3.7
97
+ signing_key:
98
+ specification_version: 3
99
+ summary: Queue Classic is an alternative queueing library for Ruby apps (Rails, Sinatra, Etc...) Queue Classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, Queue Classic only requires the pg and json.(simple)
100
+ test_files:
101
+ - test/api_test.rb
102
+ - test/durable_array_test.rb
103
+ - test/queue_test.rb
104
+ - test/worker_test.rb