queue_classic 3.0.0beta → 3.0.0rc

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ae7f3fa42909c77c861db2ef125fab2ba01205b
-  data.tar.gz: a41224c39e1966276a5a66403b705426c5d3ff63
+  metadata.gz: 549f3341849e84bc2068f82448e39e98c0ef2051
+  data.tar.gz: 0746db47bbe386bd31e348b9d925985e8b185c04
 SHA512:
-  metadata.gz: 4a0d7b6b9f80e240ce3ac76c3fd488870ca648439c66917eceec454c7595dbbac7193d17a91ea099b763f25e39f76038761521d799ae3eaf4b5f09ed5c64b995
-  data.tar.gz: 92ff9563d1b98f03b096704524059a8d3f72d4ccd2626517af62b5067784641ff4eccb4f8864a9830fdc95ec06a3d8b0e5d6cc04c47656a247c2c0b40ad1d6df
+  metadata.gz: 5d9d9f8e660d294671ef3fd6d99f96fa2e092049920f72b754c62fa59857466978e6a7f10544fd35de9e52e6d72d99fb51dadb08cc2c0b8b002906f6628fe672
+  data.tar.gz: 3174b46493945c6b126323fa20dbaa1bf53bfd40a6ff07926a934ea4820e5e67276331d872d77e182fc57127e590f916a2042b48df9e9bee63965cd4d0f20f26

data/lib/queue_classic.rb

@@ -18,7 +18,7 @@ module QC
   # notes the queue. You can point your workers
   # at different queues but only one at a time.
   QUEUE = ENV["QUEUE"] || "default"
-  QUEUES = ENV["QUEUES"] || []
+  QUEUES = (ENV["QUEUES"] && ENV["QUEUES"].split(",")) || []
 
   # Set this to 1 for strict FIFO.
   # There is nothing special about 9....

data/lib/queue_classic/queue.rb

@@ -3,6 +3,7 @@ require 'queue_classic/conn_adapter'
 require 'json'
 
 module QC
+  # The queue class maps a queue abstraction onto a database table.
   class Queue
 
     attr_reader :name, :top_bound
@@ -19,6 +20,20 @@ module QC
       @adapter ||= QC.default_conn_adapter
     end
 
+    # enqueue(m,a) inserts a row into the jobs table and trigger a notification.
+    # The job's queue is represented by a name column in the row.
+    # There is a trigger on the table which will send a NOTIFY event
+    # on a channel which corresponds to the name of the queue.
+    # The method argument is a string encoded ruby expression. The expression
+    # will be separated by a `.` character and then `eval`d.
+    # Examples of the method argument include: `puts`, `Kernel.puts`,
+    # `MyObject.new.puts`.
+    # The args argument will be encoded as JSON and stored as a JSON datatype
+    # in the row. (If the version of PG does not support JSON,
+    # then the args will be stored as text.
+    # The args are stored as a collection and then splatted inside the worker.
+    # Examples of args include: `'hello world'`, `['hello world']`,
+    # `'hello', 'world'`.
     def enqueue(method, *args)
       QC.log_yield(:measure => 'queue.enqueue') do
         s="INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
@@ -37,6 +52,13 @@ module QC
       end
     end
 
+    def unlock(id)
+      QC.log_yield(:measure => 'queue.unlock') do
+        s = "UPDATE #{TABLE_NAME} set locked_at = null where id = $1"
+        conn_adapter.execute(s, id)
+      end
+    end
+
     def delete(id)
       QC.log_yield(:measure => 'queue.delete') do
         conn_adapter.execute("DELETE FROM #{TABLE_NAME} where id = $1", id)

data/lib/queue_classic/tasks.rb

@@ -8,9 +8,22 @@ end
 namespace :qc do
   desc "Start a new worker for the (default or $QUEUE) queue"
   task :work  => :environment do
-    trap('INT') {exit}
-    trap('TERM') {@worker.stop}
     @worker = QC::Worker.new
+
+    trap('INT') do
+      $stderr.puts("Received INT. Shutting down.")
+      if !@worker.running
+        $stderr.puts("Worker has stopped running. Exit.")
+        exit(1)
+      end
+      @worker.stop
+    end
+
+    trap('TERM') do
+      $stderr.puts("Received Term. Shutting down.")
+      @worker.stop
+    end
+
     @worker.start
   end
 

data/lib/queue_classic/worker.rb

@@ -3,47 +3,63 @@ require 'queue_classic/queue'
 require 'queue_classic/conn_adapter'
 
 module QC
+  # A Worker object can process jobs from one or many queues.
   class Worker
 
     attr_accessor :queues, :running
-    # In the case no arguments are passed to the initializer,
-    # the defaults are pulled from the environment variables.
+
+    # Creates a new worker but does not start the worker. See Worker#start.
+    # This method takes a single hash argument. The following keys are read:
+    # fork_worker:: Worker forks each job execution.
+    # wait_interval:: Time to wait between failed lock attempts
+    # connection:: PGConn object.
+    # q_name:: Name of a single queue to process.
+    # q_names:: Names of queues to process. Will process left to right.
+    # top_bound:: Offset to the head of the queue. 1 == strict FIFO.
     def initialize(args={})
       @fork_worker = args[:fork_worker] || QC::FORK_WORKER
       @wait_interval = args[:wait_interval] || QC::WAIT_TIME
       @conn_adapter = ConnAdapter.new(args[:connection])
       @queues = setup_queues(@conn_adapter,
-        args[:q_name], args[:q_names], args[:top_bound])
+        (args[:q_name] || QC::QUEUE),
+        (args[:q_names] || QC::QUEUES),
+        (args[:top_bound] || QC::TOP_BOUND))
       log(args.merge(:at => "worker_initialized"))
       @running = true
     end
 
-    # Start a loop and work jobs indefinitely.
-    # Call this method to start the worker.
-    # This is the easiest way to start working jobs.
+    # Commences the working of jobs.
+    # start() spins on @running –which is initialized as true.
+    # This method is the primary entry point to starting the worker.
+    # The canonical example of starting a worker is as follows:
+    # QC::Worker.new.start
     def start
       while @running
         @fork_worker ? fork_and_work : work
       end
     end
 
-    # Call this method to stop the worker.
-    # The worker may not stop immediately if the worker
-    # is sleeping.
+    # Signals the worker to stop taking new work.
+    # This method has no immediate effect. However, there are
+    # two loops in the worker (one in #start and another in #lock_job)
+    # which check the @running variable to determine if further progress
+    # is desirable. In the case that @running is false, the aforementioned
+    # methods will short circuit and cause the blocking call to #start
+    # to unblock.
     def stop
       @running = false
     end
 
-    # This method will tell the ruby process to FORK.
-    # Define setup_child to hook into the forking process.
-    # Using setup_child is good for re-establishing database connections.
+    # Calls Worker#work but after the current process is forked.
+    # The parent process will wait on the child process to exit.
     def fork_and_work
       cpid = fork {setup_child; work}
       log(:at => :fork, :pid => cpid)
       Process.wait(cpid)
     end
 
-    # This method will lock a job & process the job.
+    # Blocks on locking a job, and once a job is locked,
+    # it will process the job.
     def work
       queue, job = lock_job
       if queue && job
@@ -54,8 +70,12 @@ module QC
     end
 
     # Attempt to lock a job in the queue's table.
-    # Return a hash when a job is locked.
-    # Caller responsible for deleting the job when finished.
+    # If a job can be locked, this method returns an array with
+    # 2 elements. The first element is the queue from which the job was locked
+    # and the second is a hash representation of the job.
+    # If a job is returned, its locked_at column has been set in the
+    # job's row. It is the caller's responsibility to delete the job row
+    # from the table when the job is complete.
     def lock_job
       log(:at => "lock_job")
       job = nil
@@ -70,17 +90,26 @@ module QC
     end
 
     # A job is processed by evaluating the target code.
-    # Errors are delegated to the handle_failure method.
-    # Also, this method will make the best attempt to delete the job
-    # from the queue before returning.
+    # if the job is evaluated with no exceptions
+    # then it is deleted from the queue.
+    # If the job has raised an exception the responsibility of what
+    # to do with the job is delegated to Worker#handle_failure.
+    # If the job is not finished and an INT signal is traped,
+    # this method will unlock the job in the queue.
     def process(queue, job)
+      finished = false
       begin
-        call(job)
+        call(job).tap do
+          queue.delete(job[:id])
+          finished = true
+        end
       rescue => e
         handle_failure(job, e)
+        finished = true
       ensure
-        queue.delete(job[:id])
-        log(:at => "delete_job", :job => job[:id])
+        if !finished
+          queue.unlock(job[:id])
+        end
       end
     end
 
@@ -89,9 +118,9 @@ module QC
     # the object and pass the args.
     def call(job)
       args = job[:args]
-      klass = eval(job[:method].split(".").first)
-      message = job[:method].split(".").last
-      klass.send(message, *args)
+      receiver_str, _, message = job[:method].rpartition('.')
+      receiver = eval(receiver_str)
+      receiver.send(message, *args)
     end
 
     # This method will be called when an exception
@@ -113,10 +142,8 @@ module QC
 
     private
 
-    def setup_queues(adapter, q_name, q_names, top_bound)
-      name = q_name || QC::QUEUE
-      names = q_names || QC::QUEUES
-      names << name unless names.include?(name)
+    def setup_queues(adapter, queue, queues, top_bound)
+      names = queues.length > 0 ? queues : [queue]
       names.map do |name|
         QC::Queue.new(name, top_bound).tap do |q|
           q.conn_adapter = adapter

data/readme.md

@@ -12,7 +12,7 @@ Features:
 * JSON data format.
 * Forking workers.
 * Workers can work multiple queues.
-* [Fuzzy-FIFO support](http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf).
+* Reduced row contention using a [relaxed FIFO](http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf) technique.
 
 Contents:
 
@@ -70,22 +70,20 @@ require 'queue_classic/tasks'
 ```
 
 Start the worker via the Rakefile.
-
 ```bash
 $ bundle exec rake qc:work
 ```
 
 Setup a worker to work a non-default queue.
-
 ```bash
 $ QUEUE="priority_queue" bundle exec rake qc:work
 ```
 
 Setup a worker to work multiple queues.
-
 ```bash
-$ QUEUE="priority_queue, secondary_queue" bundle exec rake qc:work
+$ QUEUES="priority_queue, secondary_queue" bundle exec rake qc:work
 ```
+In this scenario, on each iteration of the worker's loop, it will look for jobs in the first queue prior to looking at the second queue. This means that the first queue must be empty before the worker will look at the second queue.
 
 #### Custom Worker
 
@@ -98,9 +96,9 @@ require 'queue_classic'
 FailedQueue = QC::Queue.new("failed_jobs")
 
 class MyWorker < QC::Worker
- 	def handle_failure(job, e)
-		FailedQueue.enqueue(job)
- 	end
+  def handle_failure(job, e)
+    FailedQueue.enqueue(job[:method], *job[:args])
+  end
 end
 
 worker = MyWorker.new
@@ -109,10 +107,8 @@ trap('INT') {exit}
 trap('TERM') {worker.stop}
 
 loop do
-	job = worker.lock_job
-	Thread.new do
-	  Timeout::timeout(5) { worker.process(job) }
-	end
+  job = worker.lock_job
+  Timeout::timeout(5) { worker.process(job) }
 end
 ```
 
@@ -134,27 +130,27 @@ $ ruby -r queue_classic -e "QC::Worker.new.work"
 ### Ruby on Rails Setup
 
 Declare dependencies in Gemfile.
-
 ```ruby
 source "http://rubygems.org"
-gem "queue_classic", "2.2.3"
-```
-
-By default, queue_classic will use the QC_DATABASE_URL falling back on DATABASE_URL. The URL must be in the following format: `postgres://username:password@localhost/database_name`.  If you use Heroku's PostgreSQL service, this will already be set. If you don't want to set this variable, you can set the connection in an initializer. **QueueClassic will maintain its own connection to the database.** This may double the number of connections to your database. Set QC::Conn.connection to share the connection between Rails & QueueClassic
-
-```ruby
-require 'queue_classic'
-QC::Conn.connection = ActiveRecord::Base.connection.raw_connection
+gem "queue_classic", "3.0.0beta"
 ```
 
-Next you need to run the queue classic generator to create the database
-migration. This will setup the necessary table to use queue classic.
-
+Add the database tables and stored procedures.
 ```
 rails generate queue_classic:install
 rake db:migrate
 ```
 
+By default, queue_classic will use the QC_DATABASE_URL falling back on DATABASE_URL. The URL must be in the following format: `postgres://username:password@localhost/database_name`.  If you use Heroku's PostgreSQL service, this will already be set. If you don't want to set this variable, you can set the connection in an initializer. **QueueClassic will maintain its own connection to the database.** This may double the number of connections to your database.
+
+You can share your active record connection with queue_classic –**however this is not thread safe.**
+
+```ruby
+require 'queue_classic'
+QC.default_conn_adapter = QC::ConnAdapter.new(
+	ActiveRecord::Base.connection.raw_connection)
+```
+
 **Note on using ActiveRecord migrations:** If you use the migration, and you wish to use commands that reset the database from the stored schema (e.g. `rake db:reset`), your application must be configured with `config.active_record.schema_format = :sql` in `config/application.rb`.  If you don't do this, the PL/pgSQL function that queue_classic creates will be lost when you reset the database.
 
 ### Rake Task Setup
@@ -195,7 +191,7 @@ export DEBUG="true"
 If you think you have found a bug, feel free to open an issue. Use the following template for the new issue:
 
 1. List Versions: Ruby, PostgreSQL, queue_classic.
-2. Define what you would have expcted to happen.
+2. Define what you would have expected to happen.
 3. List what actually happened.
 4. Provide sample codes & commands which will reproduce the problem.
 

data/test/worker_test.rb

@@ -5,6 +5,7 @@ module TestObject
   def no_args; return nil; end
   def one_arg(a); return a; end
   def two_args(a,b); return [a,b]; end
+  def forty_two; OpenStruct.new(number: 42); end
 end
 
 # This not only allows me to test what happens
@@ -167,4 +168,12 @@ class WorkerTest < QCTest
     assert_equal(3, r_queue.count)
   end
 
+  def test_work_with_more_complex_construct
+    QC.enqueue("TestObject.forty_two.number")
+    worker = TestWorker.new
+    r = worker.work
+    assert_equal(42, r)
+    assert_equal(0, worker.failed_count)
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 3.0.0beta
+  version: 3.0.0rc
 platform: ruby
 authors:
 - Ryan Smith (♠ ace hacker)