que 0.5.0 → 0.6.0

data/lib/que/adapters/base.rb

@@ -1,3 +1,5 @@
+require 'time' # For Time.parse.
+
 module Que
   module Adapters
     autoload :ActiveRecord,   'que/adapters/active_record'
@@ -25,11 +27,35 @@ module Que
         false
       end
 
-      def execute(*args)
+      def execute(command, params = [])
+        params = params.map do |param|
+          case param
+            # The pg gem unfortunately doesn't convert fractions of time instances, so cast them to a string.
+            when Time then param.strftime("%Y-%m-%d %H:%M:%S.%6N %z")
+            when Array, Hash then JSON_MODULE.dump(param)
+            else param
+          end
+        end
+
+        cast_result \
+          case command
+            when Symbol then execute_prepared(command, params)
+            when String then execute_sql(command, params)
+          end
+      end
+
+      def in_transaction?
+        checkout { |conn| conn.transaction_status != ::PG::PQTRANS_IDLE }
+      end
+
+      private
+
+      def execute_sql(sql, params)
+        args = params.empty? ? [sql] : [sql, params]
         checkout { |conn| conn.async_exec(*args) }
       end
 
-      def execute_prepared(name, params = [])
+      def execute_prepared(name, params)
         checkout do |conn|
           statements = @prepared_statements[conn] ||= {}
 
@@ -42,8 +68,53 @@ module Que
         end
       end
 
-      def in_transaction?
-        checkout { |conn| conn.transaction_status != ::PG::PQTRANS_IDLE }
+      HASH_DEFAULT_PROC = proc { |hash, key| hash[key.to_s] if Symbol === key }
+
+      INDIFFERENTIATOR = proc do |object|
+        case object
+        when Array
+          object.each(&INDIFFERENTIATOR)
+        when Hash
+          object.default_proc = HASH_DEFAULT_PROC
+          object.each { |key, value| object[key] = INDIFFERENTIATOR.call(value) }
+          object
+        else
+          object
+        end
+      end
+
+      CAST_PROCS = {}
+
+      # Integer, bigint, smallint:
+      CAST_PROCS[23] = CAST_PROCS[20] = CAST_PROCS[21] = proc(&:to_i)
+
+      # Timestamp with time zone.
+      CAST_PROCS[1184] = Time.method(:parse)
+
+      # JSON.
+      CAST_PROCS[114] = JSON_MODULE.method(:load)
+
+      # Boolean:
+      CAST_PROCS[16] = 't'.method(:==)
+
+      def cast_result(result)
+        output = result.to_a
+
+        result.fields.each_with_index do |field, index|
+          if converter = CAST_PROCS[result.ftype(index)]
+            output.each do |hash|
+              unless (value = hash[field]).nil?
+                hash[field] = converter.call(value)
+              end
+            end
+          end
+        end
+
+        if result.first.respond_to?(:with_indifferent_access)
+          output.map(&:with_indifferent_access)
+        else
+          output.each(&INDIFFERENTIATOR)
+        end
       end
     end
   end

data/lib/que/job.rb

@@ -3,12 +3,11 @@ module Que
     attr_reader :attrs
 
     def initialize(attrs)
-      @attrs        = attrs
-      @attrs[:args] = Que.indifferentiate JSON_MODULE.load(@attrs[:args])
+      @attrs = attrs
     end
 
     # Subclasses should define their own run methods, but keep an empty one
-    # here so that Que::Job.queue can queue an empty job in testing.
+    # here so that Que::Job.enqueue can queue an empty job in testing.
     def run(*args)
     end
 
@@ -20,45 +19,61 @@ module Que
     private
 
     def destroy
-      Que.execute :destroy_job, attrs.values_at(:priority, :run_at, :job_id)
+      Que.execute :destroy_job, attrs.values_at(:queue, :priority, :run_at, :job_id)
       @destroyed = true
     end
 
+    @retry_interval = proc { |count| count ** 4 + 3 }
+
     class << self
-      def queue(*args)
+      attr_reader :retry_interval
+
+      def enqueue(*args)
         if args.last.is_a?(Hash)
-          options  = args.pop
-          run_at   = options.delete(:run_at)
-          priority = options.delete(:priority)
+          options   = args.pop
+          queue     = options.delete(:queue) || '' if options.key?(:queue)
+          job_class = options.delete(:job_class)
+          run_at    = options.delete(:run_at)
+          priority  = options.delete(:priority)
           args << options if options.any?
         end
 
-        attrs = {:job_class => to_s, :args => JSON_MODULE.dump(args)}
+        attrs = {:job_class => job_class || to_s, :args => args}
+
+        if t = run_at || @run_at && @run_at.call || @default_run_at && @default_run_at.call
+          attrs[:run_at] = t
+        end
 
-        if time = run_at || @default_run_at && @default_run_at.call
-          attrs[:run_at] = time
+        if p = priority || @priority || @default_priority
+          attrs[:priority] = p
         end
 
-        if pty = priority || @default_priority
-          attrs[:priority] = pty
+        if q = queue || @queue
+          attrs[:queue] = q
         end
 
-        if Que.mode == :sync && !time
-          run_job(attrs)
+        if Que.mode == :sync && !t
+          run(*attrs[:args])
         else
-          values = Que.execute(:insert_job, attrs.values_at(:priority, :run_at, :job_class, :args)).first
-          Que.adapter.wake_worker_after_commit unless time
+          values = Que.execute(:insert_job, attrs.values_at(:queue, :priority, :run_at, :job_class, :args)).first
+          Que.adapter.wake_worker_after_commit unless t
           new(values)
         end
       end
 
-      def work
+      alias queue enqueue
+
+      def run(*args)
+        new(:args => args).tap(&:_run)
+      end
+
+      def work(queue = '')
         # Since we're taking session-level advisory locks, we have to hold the
         # same connection throughout the process of getting a job, working it,
         # deleting it, and removing the lock.
         Que.adapter.checkout do
           begin
-            if job = Que.execute(:lock_job).first
+            if job = Que.execute(:lock_job, [queue]).first
               # Edge case: It's possible for the lock_job query to have
               # grabbed a job that's already been worked, if it took its MVCC
               # snapshot while the job was processing, but didn't attempt the
@@ -69,10 +84,12 @@ module Que
               # Note that there is currently no spec for this behavior, since
               # I'm not sure how to reliably commit a transaction that deletes
               # the job in a separate thread between lock_job and check_job.
-              if Que.execute(:check_job, job.values_at(:priority, :run_at, :job_id)).none?
+              if Que.execute(:check_job, job.values_at(:queue, :priority, :run_at, :job_id)).none?
                 {:event => :job_race_condition}
               else
-                {:event => :job_worked, :job => run_job(job).attrs}
+                klass = class_for(job[:job_class])
+                klass.new(job)._run
+                {:event => :job_worked, :job => job}
               end
             else
               {:event => :job_unavailable}
@@ -80,11 +97,11 @@ module Que
           rescue => error
             begin
               if job
-                # Borrowed the backoff formula and error data format from delayed_job.
-                count   = job[:error_count].to_i + 1
-                delay   = count ** 4 + 3
-                message = "#{error.message}\n#{error.backtrace.join("\n")}"
-                Que.execute :set_error, [count, delay, message] + job.values_at(:priority, :run_at, :job_id)
+                count    = job[:error_count].to_i + 1
+                interval = klass && klass.retry_interval || retry_interval
+                delay    = interval.respond_to?(:call) ? interval.call(count) : interval
+                message  = "#{error.message}\n#{error.backtrace.join("\n")}"
+                Que.execute :set_error, [count, delay, message] + job.values_at(:queue, :priority, :run_at, :job_id)
               end
             rescue
               # If we can't reach the database for some reason, too bad, but
@@ -111,8 +128,8 @@ module Que
 
       private
 
-      def run_job(attrs)
-        attrs[:job_class].split('::').inject(Object, &:const_get).new(attrs).tap(&:_run)
+      def class_for(string)
+        string.split('::').inject(Object, &:const_get)
       end
     end
   end

data/lib/que/migrations.rb

@@ -3,7 +3,7 @@ module Que
     # In order to ship a schema change, add the relevant up and down sql files
     # to the migrations directory, and bump the version both here and in the
     # add_que generator template.
-    CURRENT_VERSION = 2
+    CURRENT_VERSION = 3
 
     class << self
       def migrate!(options = {:version => CURRENT_VERSION})
@@ -21,7 +21,8 @@ module Que
           end
 
           steps.each do |step|
-            Que.execute File.read("#{File.dirname(__FILE__)}/migrations/#{step}-#{direction}.sql")
+            sql = File.read("#{File.dirname(__FILE__)}/migrations/#{step}/#{direction}.sql")
+            Que.execute(sql)
           end
 
           set_db_version(version)

data/lib/que/migrations/3/down.sql

@@ -0,0 +1,5 @@
+ALTER TABLE que_jobs
+  DROP CONSTRAINT que_jobs_pkey,
+  DROP COLUMN queue,
+  ALTER COLUMN priority TYPE integer,
+  ADD CONSTRAINT que_jobs_pkey PRIMARY KEY (priority, run_at, job_id);

data/lib/que/migrations/3/up.sql

@@ -0,0 +1,5 @@
+ALTER TABLE que_jobs
+  DROP CONSTRAINT que_jobs_pkey,
+  ALTER COLUMN priority TYPE smallint,
+  ADD COLUMN queue TEXT NOT NULL DEFAULT '',
+  ADD CONSTRAINT que_jobs_pkey PRIMARY KEY (queue, priority, run_at, job_id);

data/lib/que/sql.rb

@@ -7,7 +7,8 @@ module Que
         FROM (
           SELECT j
           FROM que_jobs AS j
-          WHERE run_at <= now()
+          WHERE queue = $1::text
+          AND run_at <= now()
           ORDER BY priority, run_at, job_id
           LIMIT 1
         ) AS t1
@@ -17,7 +18,9 @@ module Que
             SELECT (
               SELECT j
               FROM que_jobs AS j
-              WHERE run_at <= now() AND (priority, run_at, job_id) > (job.priority, job.run_at, job.job_id)
+              WHERE queue = $1::text
+              AND run_at <= now()
+              AND (priority, run_at, job_id) > (job.priority, job.run_at, job.job_id)
               ORDER BY priority, run_at, job_id
               LIMIT 1
             ) AS j
@@ -27,7 +30,7 @@ module Que
           ) AS t1
         )
       )
-      SELECT priority, run_at, job_id, job_class, args, error_count
+      SELECT queue, priority, run_at, job_id, job_class, args, error_count
       FROM job
       WHERE locked
       LIMIT 1
@@ -36,9 +39,10 @@ module Que
     :check_job => %{
       SELECT 1 AS one
       FROM   que_jobs
-      WHERE  priority = $1::integer
-      AND    run_at   = $2::timestamptz
-      AND    job_id   = $3::bigint
+      WHERE  queue    = $1::text
+      AND    priority = $2::smallint
+      AND    run_at   = $3::timestamptz
+      AND    job_id   = $4::bigint
     }.freeze,
 
     :set_error => %{
@@ -46,28 +50,31 @@ module Que
       SET error_count = $1::integer,
           run_at      = now() + $2::integer * '1 second'::interval,
           last_error  = $3::text
-      WHERE priority  = $4::integer
-      AND   run_at    = $5::timestamptz
-      AND   job_id    = $6::bigint
+      WHERE queue     = $4::text
+      AND   priority  = $5::smallint
+      AND   run_at    = $6::timestamptz
+      AND   job_id    = $7::bigint
     }.freeze,
 
     :insert_job => %{
       INSERT INTO que_jobs
-      (priority, run_at, job_class, args)
+      (queue, priority, run_at, job_class, args)
       VALUES
-      (coalesce($1, 100)::integer, coalesce($2, 'now')::timestamptz, $3::text, coalesce($4, '[]')::json)
+      (coalesce($1, '')::text, coalesce($2, 100)::smallint, coalesce($3, 'now')::timestamptz, $4::text, coalesce($5, '[]')::json)
       RETURNING *
     }.freeze,
 
     :destroy_job => %{
       DELETE FROM que_jobs
-      WHERE priority = $1::integer
-      AND   run_at   = $2::timestamptz
-      AND   job_id   = $3::bigint
+      WHERE queue    = $1::text
+      AND   priority = $2::smallint
+      AND   run_at   = $3::timestamptz
+      AND   job_id   = $4::bigint
     }.freeze,
 
     :job_stats => %{
-      SELECT job_class,
+      SELECT queue,
+             job_class,
              count(*)                    AS count,
              count(locks.job_id)         AS count_working,
              sum((error_count > 0)::int) AS count_errored,
@@ -79,7 +86,7 @@ module Que
         FROM pg_locks
         WHERE locktype = 'advisory'
       ) locks USING (job_id)
-      GROUP BY job_class
+      GROUP BY queue, job_class
       ORDER BY count(*) DESC
     }.freeze,
 
@@ -100,5 +107,5 @@ module Que
         WHERE locktype = 'advisory'
       ) pg USING (job_id)
     }.freeze
-  }
+  }.freeze
 end

data/lib/que/version.rb

@@ -1,3 +1,3 @@
 module Que
-  Version = '0.5.0'
+  Version = '0.6.0'
 end

data/lib/que/worker.rb

@@ -8,10 +8,11 @@ module Que
     # synchronize access to it.
     include MonitorMixin
 
-    attr_reader :thread, :state
+    attr_reader :thread, :state, :queue
 
-    def initialize
-      super # For MonitorMixin.
+    def initialize(queue = '')
+      super() # For MonitorMixin.
+      @queue  = queue
       @state  = :working
       @thread = Thread.new { work_loop }
     end
@@ -71,7 +72,7 @@ module Que
       loop do
         time   = Time.now
         cycle  = nil
-        result = Job.work
+        result = Job.work(queue)
 
         case result[:event]
         when :job_unavailable
@@ -167,7 +168,7 @@ module Que
           Que.log :event => 'worker_count_change', :value => count.to_s
 
           if count > worker_count
-            workers.push *(count - worker_count).times.map{new}
+            workers.push *(count - worker_count).times.map{new(ENV['QUE_QUEUE'] || '')}
           elsif count < worker_count
             workers.pop(worker_count - count).each(&:stop).each(&:wait_until_stopped)
           end

data/spec/adapters/active_record_spec.rb

@@ -22,7 +22,7 @@ unless defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
           end
         end
 
-        ActiveRecordJob.queue
+        ActiveRecordJob.enqueue
         Que::Job.work
 
         $pid1.should == $pid2
@@ -32,7 +32,7 @@ unless defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
     end
 
     it "should instantiate args as ActiveSupport::HashWithIndifferentAccess" do
-      ArgsJob.queue :param => 2
+      ArgsJob.enqueue :param => 2
       Que::Job.work
       $passed_args.first[:param].should == 2
       $passed_args.first.should be_an_instance_of ActiveSupport::HashWithIndifferentAccess
@@ -42,7 +42,7 @@ unless defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
       Que.mode = :async
       sleep_until { Que::Worker.workers.all? &:sleeping? }
 
-      Que::Job.queue :run_at => 1.minute.ago
+      Que::Job.enqueue :run_at => 1.minute.ago
       DB[:que_jobs].get(:run_at).should be_within(3).of Time.now - 60
 
       Que.wake_interval = 0.005.seconds
@@ -54,17 +54,17 @@ unless defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
       sleep_until { Que::Worker.workers.all? &:sleeping? }
 
       # Wakes a worker immediately when not in a transaction.
-      Que::Job.queue
+      Que::Job.enqueue
       sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
 
       ActiveRecord::Base.transaction do
-        Que::Job.queue
+        Que::Job.enqueue
         Que::Worker.workers.each { |worker| worker.should be_sleeping }
       end
       sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
 
       # Do nothing when queueing with a specific :run_at.
-      BlockJob.queue :run_at => Time.now
+      BlockJob.enqueue :run_at => Time.now
       Que::Worker.workers.each { |worker| worker.should be_sleeping }
     end