que 0.3.0 → 0.4.0

data/lib/que.rb

@@ -39,19 +39,47 @@ module Que
       execute "DELETE FROM que_jobs"
     end
 
+    def job_stats
+      execute :job_stats
+    end
+
+    def worker_states
+      execute :worker_states
+    end
+
     def execute(command, *args)
-      case command
-        when Symbol then adapter.execute_prepared(command, *args)
-        when String then adapter.execute(command, *args)
-      end
+      indifferentiate case command
+                        when Symbol then adapter.execute_prepared(command, *args)
+                        when String then adapter.execute(command, *args)
+                      end.to_a
     end
 
     def log(level, text)
       logger.send level, "[Que] #{text}" if logger
     end
 
+    # Helper for making hashes indifferently-accessible, even when nested
+    # within each other and within arrays.
+    def indifferentiate(object)
+      case object
+      when Hash
+        h = if {}.respond_to?(:with_indifferent_access) # Better support for Rails.
+              {}.with_indifferent_access
+            else
+              Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
+            end
+
+        object.each { |k, v| h[k] = indifferentiate(v) }
+        h
+      when Array
+        object.map { |v| indifferentiate(v) }
+      else
+        object
+      end
+    end
+
     # Copy some of the Worker class' config methods here for convenience.
-    [:mode, :mode=, :worker_count=, :sleep_period, :sleep_period=, :stop!].each do |meth|
+    [:mode, :mode=, :worker_count, :worker_count=, :wake_interval, :wake_interval=, :stop!, :wake!, :wake_all!].each do |meth|
       define_method(meth) { |*args| Worker.send(meth, *args) }
     end
   end

data/lib/que/adapters/base.rb

@@ -12,11 +12,19 @@ module Que
 
       # The only method that adapters really need to implement. Should lock a
       # PG::Connection (or something that acts like a PG::Connection) so that
-      # no other threads are using it and yield it to the block.
+      # no other threads are using it and yield it to the block. Should also
+      # be re-entrant.
       def checkout(&block)
         raise NotImplementedError
       end
 
+      # Called after a job is queued in async mode, to prompts a worker to
+      # wake up after the current transaction commits. Not all adapters will
+      # implement this.
+      def wake_worker_after_commit
+        false
+      end
+
       def execute(*args)
         checkout { |conn| conn.async_exec(*args) }
       end

data/lib/que/adapters/pg.rb

@@ -3,6 +3,8 @@ require 'monitor'
 module Que
   module Adapters
     class PG < Base
+      attr_reader :lock
+
       def initialize(pg)
         @pg   = pg
         @lock = Monitor.new # Must be re-entrant.

data/lib/que/adapters/sequel.rb

@@ -9,6 +9,10 @@ module Que
       def checkout(&block)
         @db.synchronize(&block)
       end
+
+      def wake_worker_after_commit
+        @db.after_commit { Que.wake! }
+      end
     end
   end
 end

data/lib/que/job.rb

@@ -2,8 +2,11 @@ require 'multi_json'
 
 module Que
   class Job
+    attr_reader :attrs
+
     def initialize(attrs)
-      @attrs = attrs
+      @attrs        = attrs
+      @attrs[:args] = Que.indifferentiate MultiJson.load(@attrs[:args])
     end
 
     # Subclasses should define their own run methods, but keep an empty one
@@ -12,18 +15,16 @@ module Que
     end
 
     def _run
-      start = Time.now
-
-      run *@attrs[:args]
+      time = Time.now
+      run *attrs[:args]
       destroy unless @destroyed
-
-      Que.log :info, "Worked job in #{((Time.now - start) * 1000).round(1)} ms: #{inspect}"
+      Que.log :info, "Worked job in #{((Time.now - time) * 1000).round(1)} ms: #{inspect}"
     end
 
     private
 
     def destroy
-      Que.execute :destroy_job, [@attrs[:priority], @attrs[:run_at], @attrs[:job_id]]
+      Que.execute :destroy_job, attrs.values_at(:priority, :run_at, :job_id)
       @destroyed = true
     end
 
@@ -38,18 +39,20 @@ module Que
 
         attrs = {:job_class => to_s, :args => MultiJson.dump(args)}
 
-        if t = run_at || @default_run_at && @default_run_at.call
-          attrs[:run_at] = t
+        if time = run_at || @default_run_at && @default_run_at.call
+          attrs[:run_at] = time
         end
 
-        if p = priority || @default_priority
-          attrs[:priority] = p
+        if pty = priority || @default_priority
+          attrs[:priority] = pty
         end
 
-        if Que.mode == :sync && !attrs[:run_at]
+        if Que.mode == :sync && !time
           run_job(attrs)
         else
-          Que.execute *insert_sql(attrs)
+          values = Que.execute(:insert_job, attrs.values_at(:priority, :run_at, :job_class, :args)).first
+          Que.adapter.wake_worker_after_commit unless time
+          new(values)
         end
       end
 
@@ -65,7 +68,7 @@ module Que
         # deleting it, and removing the lock.
         Que.adapter.checkout do
           begin
-            if row = Que.execute(:lock_job).first
+            if job = Que.execute(:lock_job).first
               # Edge case: It's possible for the lock_job query to have
               # grabbed a job that's already been worked, if it took its MVCC
               # snapshot while the job was processing, but didn't attempt the
@@ -76,21 +79,21 @@ module Que
               # Note that there is currently no spec for this behavior, since
               # I'm not sure how to reliably commit a transaction that deletes
               # the job in a separate thread between lock_job and check_job.
-              return true if Que.execute(:check_job, [row['priority'], row['run_at'], row['job_id']]).none?
+              return true if Que.execute(:check_job, job.values_at(:priority, :run_at, :job_id)).none?
 
-              run_job(row)
+              run_job(job)
             else
               Que.log :info, "No jobs available..."
               nil
             end
           rescue => error
             begin
-              if row
+              if job
                 # Borrowed the backoff formula and error data format from delayed_job.
-                count   = row['error_count'].to_i + 1
-                run_at  = count ** 4 + 3
+                count   = job[:error_count].to_i + 1
+                delay   = count ** 4 + 3
                 message = "#{error.message}\n#{error.backtrace.join("\n")}"
-                Que.execute :set_error, [count, run_at, message, row['priority'], row['run_at'], row['job_id']]
+                Que.execute :set_error, [count, delay, message] + job.values_at(:priority, :run_at, :job_id)
               end
             rescue
               # If we can't reach the database for some reason, too bad, but
@@ -110,57 +113,20 @@ module Que
             return !error.is_a?(PG::Error)
           ensure
             # Clear the advisory lock we took when locking the job. Important
-            # to do this so that they don't pile up in the database.
-            Que.execute "SELECT pg_advisory_unlock_all()" if row
+            # to do this so that they don't pile up in the database. Again, if
+            # we can't reach the database, don't crash the work loop.
+            begin
+              Que.execute "SELECT pg_advisory_unlock($1)", [job[:job_id]] if job
+            rescue
+            end
           end
         end
       end
 
       private
 
-      # Column names are not escaped, so this method should not be called with untrusted hashes.
-      def insert_sql(hash)
-        number       = 0
-        columns      = []
-        placeholders = []
-        values       = []
-
-        hash.each do |key, value|
-          columns      << key
-          placeholders << "$#{number += 1}"
-          values       << value
-        end
-
-        ["INSERT INTO que_jobs (#{columns.join(', ')}) VALUES (#{placeholders.join(', ')})", values]
-      end
-
       def run_job(attrs)
-        attrs = indifferentiate(attrs)
-        attrs[:args] = indifferentiate(MultiJson.load(attrs[:args]))
-        klass = attrs[:job_class].split('::').inject(Object, &:const_get)
-        klass.new(attrs).tap(&:_run)
-      end
-
-      def indifferentiate(input)
-        case input
-        when Hash
-          h = indifferent_hash
-          input.each { |k, v| h[k] = indifferentiate(v) }
-          h
-        when Array
-          input.map { |v| indifferentiate(v) }
-        else
-          input
-        end
-      end
-
-      def indifferent_hash
-        # Tiny hack to better support Rails.
-        if {}.respond_to?(:with_indifferent_access)
-          {}.with_indifferent_access
-        else
-          Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
-        end
+        attrs[:job_class].split('::').inject(Object, &:const_get).new(attrs).tap(&:_run)
       end
     end
   end

data/lib/que/rake_tasks.rb

@@ -4,7 +4,6 @@ namespace :que do
     require 'logger'
 
     Que.logger       = Logger.new(STDOUT)
-    Que.mode         = :async
     Que.worker_count = (ENV['WORKER_COUNT'] || 4).to_i
 
     # When changing how signals are caught, be sure to test the behavior with

data/lib/que/sql.rb

@@ -1,87 +1,132 @@
 module Que
   SQL = {
-    # Thanks to RhodiumToad in #postgresql for the job lock CTE. It was
-    # modified only slightly from his design.
-    :lock_job => (
-      <<-SQL
-        WITH RECURSIVE cte AS (
-          SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
+    # Thanks to RhodiumToad in #postgresql for help with the job lock CTE.
+    :lock_job => %{
+      WITH RECURSIVE job AS (
+        SELECT (j).*, pg_try_advisory_lock((j).job_id) AS locked
+        FROM (
+          SELECT j
+          FROM que_jobs AS j
+          WHERE run_at <= now()
+          ORDER BY priority, run_at, job_id
+          LIMIT 1
+        ) AS t1
+        UNION ALL (
+          SELECT (j).*, pg_try_advisory_lock((j).job_id) AS locked
           FROM (
-            SELECT job
-            FROM que_jobs AS job
-            WHERE run_at <= now()
-            ORDER BY priority, run_at, job_id
+            SELECT (
+              SELECT j
+              FROM que_jobs AS j
+              WHERE run_at <= now() AND (priority, run_at, job_id) > (job.priority, job.run_at, job.job_id)
+              ORDER BY priority, run_at, job_id
+              LIMIT 1
+            ) AS j
+            FROM job
+            WHERE NOT job.locked
             LIMIT 1
           ) AS t1
-          UNION ALL (
-            SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
-            FROM (
-              SELECT (
-               SELECT job
-                FROM que_jobs AS job
-                WHERE run_at <= now() AND (priority, run_at, job_id) > (cte.priority, cte.run_at, cte.job_id)
-                ORDER BY priority, run_at, job_id
-                LIMIT 1
-              ) AS job
-              FROM cte
-              WHERE NOT cte.locked
-              LIMIT 1
-            ) AS t1
-          )
         )
-        SELECT priority, run_at, job_id, job_class, args, error_count
-        FROM cte
-        WHERE locked
-        LIMIT 1
-      SQL
-    ).freeze,
+      )
+      SELECT priority, run_at, job_id, job_class, args, error_count
+      FROM job
+      WHERE locked
+      LIMIT 1
+    }.freeze,
 
-    :check_job => (
-      <<-SQL
-        SELECT 1 AS one
-        FROM   que_jobs
-        WHERE  priority = $1::integer
-        AND    run_at   = $2::timestamptz
-        AND    job_id   = $3::bigint
-      SQL
-    ).freeze,
+    :check_job => %{
+      SELECT 1 AS one
+      FROM   que_jobs
+      WHERE  priority = $1::integer
+      AND    run_at   = $2::timestamptz
+      AND    job_id   = $3::bigint
+    }.freeze,
 
-    :set_error => (
-      <<-SQL
-        UPDATE que_jobs
-        SET error_count = $1::integer,
-            run_at      = now() + $2::integer * '1 second'::interval,
-            last_error  = $3::text
-        WHERE priority  = $4::integer
-        AND   run_at    = $5::timestamptz
-        AND   job_id    = $6::bigint
-      SQL
-    ).freeze,
+    :set_error => %{
+      UPDATE que_jobs
+      SET error_count = $1::integer,
+          run_at      = now() + $2::integer * '1 second'::interval,
+          last_error  = $3::text
+      WHERE priority  = $4::integer
+      AND   run_at    = $5::timestamptz
+      AND   job_id    = $6::bigint
+    }.freeze,
 
-    :destroy_job => (
-      <<-SQL
-        DELETE FROM que_jobs
-        WHERE priority = $1::integer
-        AND   run_at   = $2::timestamptz
-        AND   job_id   = $3::bigint
-      SQL
-    ).freeze,
+    :insert_job => %{
+      INSERT INTO que_jobs
+      (priority, run_at, job_class, args)
+      VALUES
+      (coalesce($1, 1)::integer, coalesce($2, 'now')::timestamptz, $3::text, coalesce($4, '[]')::json)
+      RETURNING *
+    }.freeze,
 
-    :create_table => (
-      <<-SQL
-        CREATE TABLE que_jobs
-        (
-          priority    integer     NOT NULL DEFAULT 1,
-          run_at      timestamptz NOT NULL DEFAULT now(),
-          job_id      bigserial   NOT NULL,
-          job_class   text        NOT NULL,
-          args        json        NOT NULL DEFAULT '[]'::json,
-          error_count integer     NOT NULL DEFAULT 0,
-          last_error  text,
+    :destroy_job => %{
+      DELETE FROM que_jobs
+      WHERE priority = $1::integer
+      AND   run_at   = $2::timestamptz
+      AND   job_id   = $3::bigint
+    }.freeze,
 
-          CONSTRAINT que_jobs_pkey PRIMARY KEY (priority, run_at, job_id)
-        )
-      SQL
-    ).freeze
+    :job_stats => %{
+      SELECT job_class,
+             count(*)                    AS count,
+             count(locks.job_id)         AS count_working,
+             sum((error_count > 0)::int) AS count_errored,
+             max(error_count)            AS highest_error_count,
+             min(run_at)                 AS oldest_run_at
+      FROM que_jobs
+      LEFT JOIN (
+        SELECT (classid::bigint << 32) + objid::bigint AS job_id
+        FROM pg_locks
+        WHERE locktype = 'advisory'
+      ) locks USING (job_id)
+      GROUP BY job_class
+      ORDER BY count(*) DESC
+    }.freeze,
+
+    :worker_states => %{
+      SELECT que_jobs.*,
+             pg.pid          AS pg_backend_pid,
+             pg.state        AS pg_state,
+             pg.state_change AS pg_state_changed_at,
+             pg.query        AS pg_last_query,
+             pg.query_start  AS pg_last_query_started_at,
+             pg.xact_start   AS pg_transaction_started_at,
+             pg.waiting      AS pg_waiting_on_lock
+      FROM que_jobs
+      JOIN (
+        SELECT (classid::bigint << 32) + objid::bigint AS job_id, pg_stat_activity.*
+        FROM pg_locks
+        JOIN pg_stat_activity USING (pid)
+        WHERE locktype = 'advisory'
+      ) pg USING (job_id)
+    }.freeze,
+
+    :create_table => %{
+      CREATE TABLE que_jobs
+      (
+        priority    integer     NOT NULL DEFAULT 1,
+        run_at      timestamptz NOT NULL DEFAULT now(),
+        job_id      bigserial   NOT NULL,
+        job_class   text        NOT NULL,
+        args        json        NOT NULL DEFAULT '[]'::json,
+        error_count integer     NOT NULL DEFAULT 0,
+        last_error  text,
+
+        CONSTRAINT que_jobs_pkey PRIMARY KEY (priority, run_at, job_id)
+      )
+    }.freeze
+
+    # Note: if schema changes to the que_jobs table become necessary later on,
+    # a simple versioning scheme would be:
+
+    # Set version:
+    # COMMENT ON TABLE que_jobs IS '2'
+
+    # Get version:
+    # SELECT description
+    # FROM pg_description
+    # JOIN pg_class
+    # ON pg_description.objoid = pg_class.oid
+    # WHERE relname = 'que_jobs'
   }
 end