que 0.11.3 → 2.2.0

data/lib/que/poller.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+module Que
+  class Poller
+    # The following SQL statement locks a batch of jobs using a Postgres
+    # recursive CTE [1].
+    #
+    # As noted by the Postgres documentation, it may be slightly easier to
+    # think about this expression as iteration rather than recursion, despite
+    # the `RECURSION` nomenclature defined by the SQL standards committee.
+    # Recursion is used here so that jobs in the table can be iterated one-by-
+    # one until a lock can be acquired, where a non-recursive `SELECT` would
+    # have the undesirable side-effect of locking jobs unnecessarily. For
+    # example, the following might lock more than five jobs during execution:
+    #
+    #   SELECT (j).*, pg_try_advisory_lock((j).id) AS locked
+    #   FROM public.que_jobs AS j
+    #   LIMIT 5;
+    #
+    # The CTE will initially produce an "anchor" from the non-recursive term
+    # (i.e. before the `UNION`), and then use it as the contents of the
+    # working table as it continues to iterate through `que_jobs` looking for
+    # locks. The jobs table has an index on (priority, run_at, id) which
+    # allows it to walk the jobs table in a stable manner. As noted above, the
+    # recursion examines/locks one job at a time. Every time the recursive
+    # entry runs, it's output becomes the new contents of the working table,
+    # and what was previously in the working table is appended to the final
+    # result set. For more information on the basic workings of recursive
+    # CTEs, see http://www.postgresql.org/docs/devel/static/queries-with.html
+    #
+    # The polling query is provided a JSONB hash of desired job priorities.
+    # For example, if the locker has three workers free that can work a
+    # priority less than 5, and two workers free that can work a priority less
+    # than 10, the provided priority document is `{"5":3,"10":2}`. The query
+    # uses this information to decide what jobs to lock - if only high-
+    # priority workers were available, it wouldn't make sense to retrieve low-
+    # priority jobs.
+    #
+    # As each job is retrieved from the table, it is passed to
+    # lock_and_update_priorities() (which, for future flexibility, we define
+    # as a temporary function attached to the connection rather than embedding
+    # permanently into the DB schema). lock_and_update_priorities() attempts
+    # to lock the given job and, if it is able to, updates the priorities
+    # document to reflect that a job was available for that given priority.
+    # When the priorities document is emptied (all the counts of desired jobs
+    # for the various priorities have reached zero and been removed), the
+    # recursive query returns an empty result and the recursion breaks. This
+    # also happens if there aren't enough appropriate jobs in the jobs table.
+    #
+    # Also note the use of JOIN LATERAL to combine the job data with the
+    # output of lock_and_update_priorities(). The naive approach would be to
+    # write the SELECT as `SELECT (j).*, (lock_and_update_priorities(..., j)).*`,
+    # but the asterisk-expansion of the latter composite row causes the function
+    # to be evaluated twice, and to thereby take the advisory lock twice,
+    # which complicates the later unlocking step.
+    #
+    # Thanks to RhodiumToad in #postgresql for help with the original
+    # (simpler) version of the recursive job lock CTE.
+
+    SQL[:poll_jobs] =
+      %{
+        WITH RECURSIVE jobs AS (
+          SELECT
+            (j).*,
+            l.locked,
+            l.remaining_priorities
+          FROM (
+            SELECT j
+            FROM public.que_jobs AS j
+            WHERE queue = $1::text
+              AND job_schema_version = #{Que.job_schema_version}
+              AND NOT id = ANY($2::bigint[])
+              AND priority <= pg_temp.que_highest_remaining_priority($3::jsonb)
+              AND run_at <= now()
+              AND finished_at IS NULL AND expired_at IS NULL
+            ORDER BY priority, run_at, id
+            LIMIT 1
+          ) AS t1
+          JOIN LATERAL (SELECT * FROM pg_temp.lock_and_update_priorities($3::jsonb, j)) AS l ON true
+          UNION ALL (
+            SELECT
+              (j).*,
+              l.locked,
+              l.remaining_priorities
+            FROM (
+              SELECT
+                remaining_priorities,
+                (
+                  SELECT j
+                  FROM public.que_jobs AS j
+                  WHERE queue = $1::text
+                    AND job_schema_version = #{Que.job_schema_version}
+                    AND NOT id = ANY($2::bigint[])
+                    AND priority <= pg_temp.que_highest_remaining_priority(jobs.remaining_priorities)
+                    AND run_at <= now()
+                    AND finished_at IS NULL AND expired_at IS NULL
+                    AND (priority, run_at, id) >
+                      (jobs.priority, jobs.run_at, jobs.id)
+                  ORDER BY priority, run_at, id
+                  LIMIT 1
+                ) AS j
+
+              FROM jobs
+              WHERE jobs.id IS NOT NULL AND jobs.remaining_priorities != '{}'::jsonb
+              LIMIT 1
+            ) AS t1
+            JOIN LATERAL (SELECT * FROM pg_temp.lock_and_update_priorities(remaining_priorities, j)) AS l ON true
+          )
+        )
+        SELECT *
+        FROM jobs
+        WHERE locked
+      }
+
+    attr_reader \
+      :connection,
+      :queue,
+      :poll_interval,
+      :last_polled_at,
+      :last_poll_satisfied
+
+    def initialize(
+      connection:,
+      queue:,
+      poll_interval:
+    )
+      @connection          = connection
+      @queue               = queue
+      @poll_interval       = poll_interval
+      @last_polled_at      = nil
+      @last_poll_satisfied = nil
+
+      Que.internal_log :poller_instantiate, self do
+        {
+          backend_pid:   connection.backend_pid,
+          queue:         queue,
+          poll_interval: poll_interval,
+        }
+      end
+    end
+
+    def poll(
+      priorities:,
+      held_locks:
+    )
+
+      return unless should_poll?
+
+      jobs =
+        connection.execute_prepared(
+          :poll_jobs,
+          [
+            @queue,
+            "{#{held_locks.to_a.join(',')}}",
+            JSON.dump(priorities),
+          ]
+        )
+
+      @last_polled_at      = Time.now
+      @last_poll_satisfied = poll_satisfied?(priorities, jobs)
+
+      Que.internal_log :poller_polled, self do
+        {
+          queue:        @queue,
+          locked:       jobs.count,
+          priorities:   priorities,
+          held_locks:   held_locks.to_a,
+          newly_locked: jobs.map { |key| key.fetch(:id) },
+        }
+      end
+
+      jobs.map! { |job| Metajob.new(job) }
+    end
+
+    def should_poll?
+      # Never polled before?
+      last_poll_satisfied.nil? ||
+      # Plenty of jobs were available last time?
+      last_poll_satisfied == true ||
+      poll_interval_elapsed?
+    end
+
+    def poll_interval_elapsed?
+      return unless interval = poll_interval
+      (Time.now - last_polled_at) > interval
+    end
+
+    class << self
+      # Manage some temporary infrastructure (specific to the connection) that
+      # we'll use for polling. These could easily be created permanently in a
+      # migration, but that'd require another migration if we wanted to tweak
+      # them later.
+
+      def setup(connection)
+        connection.execute <<-SQL
+          -- Temporary composite type we need for our queries to work.
+          CREATE TYPE pg_temp.que_query_result AS (
+            locked boolean,
+            remaining_priorities jsonb
+          );
+
+          CREATE FUNCTION pg_temp.lock_and_update_priorities(priorities jsonb, job que_jobs)
+          RETURNS pg_temp.que_query_result
+          AS $$
+            WITH
+              -- Take the lock in a CTE because we want to use the result
+              -- multiple times while only taking the lock once.
+              lock_taken AS (
+                SELECT pg_try_advisory_lock((job).id) AS taken
+              ),
+              relevant AS (
+                SELECT priority, count
+                FROM (
+                  SELECT
+                    key::smallint AS priority,
+                    value::text::integer AS count
+                  FROM jsonb_each(priorities)
+                  ) t1
+                WHERE priority >= (job).priority
+                ORDER BY priority ASC
+                LIMIT 1
+              )
+            SELECT
+              (SELECT taken FROM lock_taken), -- R
+              CASE (SELECT taken FROM lock_taken)
+              WHEN false THEN
+                -- Simple case - we couldn't lock the job, so don't update the
+                -- priorities hash.
+                priorities
+              WHEN true THEN
+                CASE count
+                WHEN 1 THEN
+                  -- Remove the priority from the JSONB doc entirely, rather
+                  -- than leaving a zero entry in it.
+                  priorities - priority::text
+                ELSE
+                  -- Decrement the value in the JSONB doc.
+                  jsonb_set(
+                    priorities,
+                    ARRAY[priority::text],
+                    to_jsonb(count - 1)
+                  )
+                END
+              END
+            FROM relevant
+          $$
+          STABLE
+          LANGUAGE SQL;
+
+          CREATE FUNCTION pg_temp.que_highest_remaining_priority(priorities jsonb) RETURNS smallint AS $$
+            SELECT max(key::smallint) FROM jsonb_each(priorities)
+          $$
+          STABLE
+          LANGUAGE SQL;
+        SQL
+      end
+
+      def cleanup(connection)
+        connection.execute <<-SQL
+          DROP FUNCTION pg_temp.que_highest_remaining_priority(jsonb);
+          DROP FUNCTION pg_temp.lock_and_update_priorities(jsonb, que_jobs);
+          DROP TYPE pg_temp.que_query_result;
+        SQL
+      end
+    end
+
+    private
+
+    def poll_satisfied?(priorities, jobs)
+      lowest_priority = priorities.keys.max
+      jobs.count >= priorities[lowest_priority]
+    end
+  end
+end

data/lib/que/rails/railtie.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Que
+  module Rails
+    class Railtie < ::Rails::Railtie
+      config.que = Que
+
+      Que.logger     = proc { ::Rails.logger }
+      Que.connection = ::ActiveRecord if defined? ::ActiveRecord
+    end
+  end
+end

data/lib/que/result_queue.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# A thread-safe queue that holds ids for jobs that have been worked. Allows
+# appending single/retrieving all ids in a thread-safe fashion.
+
+module Que
+  class ResultQueue
+    def initialize
+      @array = []
+      @mutex = Mutex.new
+    end
+
+    def push(item)
+      sync { @array.push(item) }
+    end
+
+    def clear
+      sync { @array.pop(@array.size) }
+    end
+
+    def to_a
+      sync { @array.dup }
+    end
+
+    def length
+      sync { @array.length }
+    end
+
+    private
+
+    def sync(&block)
+      @mutex.synchronize(&block)
+    end
+  end
+end

data/lib/que/sequel/model.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+::Sequel.extension :pg_json_ops
+
+module Que
+  module Sequel
+    QUALIFIED_TABLE = ::Sequel.qualify(:public, :que_jobs)
+
+    class Model < ::Sequel::Model(QUALIFIED_TABLE)
+      dataset_module do
+        conditions = {
+          errored:   QUALIFIED_TABLE[:error_count] > 0,
+          expired:   QUALIFIED_TABLE[:expired_at] !~ nil,
+          finished:  QUALIFIED_TABLE[:finished_at] !~ nil,
+          scheduled: QUALIFIED_TABLE[:run_at] > ::Sequel::CURRENT_TIMESTAMP,
+        }
+
+        conditions.each do |name, condition|
+          subset           name,  condition
+          subset :"not_#{name}", ~condition
+        end
+
+        subset :ready,     conditions.values.map(&:~).inject(:&)
+        subset :not_ready, conditions.values.         inject(:|)
+
+        def by_job_class(job_class)
+          job_class = job_class.name if job_class.is_a?(Class)
+          where(
+            (QUALIFIED_TABLE[:job_class] =~ job_class) |
+              (QUALIFIED_TABLE[:job_class] =~ "ActiveJob::QueueAdapters::QueAdapter::JobWrapper") &
+              (QUALIFIED_TABLE[:args].pg_jsonb[0].get_text("job_class") =~ job_class)
+          )
+        end
+
+        def by_queue(queue)
+          where(QUALIFIED_TABLE[:queue] => queue)
+        end
+
+        def by_tag(tag)
+          where(QUALIFIED_TABLE[:data].pg_jsonb.contains(JSON.dump(tags: [tag])))
+        end
+
+        def by_args(*args, **kwargs)
+          where(
+            QUALIFIED_TABLE[:args].pg_jsonb.contains(JSON.dump(args)) &
+            QUALIFIED_TABLE[:kwargs].pg_jsonb.contains(JSON.dump(kwargs))
+          ) 
+        end
+      end
+    end
+  end
+end

data/lib/que/utils/assertions.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# Assertion helpers. Que has a fair amount of internal state, and there's no
+# telling what users will try to throw at it, so for ease of debugging issues it
+# makes sense to sanity-check frequently.
+
+module Que
+  module Utils
+    module Assertions
+      class AssertionFailed < Error; end
+
+      def assert(*args)
+        comparison, object, pass = _check_assertion_args(*args)
+        return object if pass
+
+        message =
+          if block_given?
+            yield.to_s
+          elsif comparison
+            "Expected #{comparison.inspect}, got #{object.inspect}!"
+          else
+            "Assertion failed!"
+          end
+
+        # Remove this method from the backtrace, to make errors clearer.
+        raise AssertionFailed, message, caller
+      end
+
+      def assert?(*args)
+        _, _, pass = _check_assertion_args(*args)
+        !!pass
+      end
+
+      private
+
+      # Want to support:
+      #   assert(x)                       # Truthiness.
+      #   assert(thing, other)            # Trip-equals.
+      #   assert([thing1, thing2], other) # Multiple Trip-equals.
+      def _check_assertion_args(first, second = (second_omitted = true; nil))
+        if second_omitted
+          comparison = nil
+          object     = first
+        else
+          comparison = first
+          object     = second
+        end
+
+        pass =
+          if second_omitted
+            object
+          elsif comparison.is_a?(Array)
+            comparison.any? { |k| k === object }
+          else
+            comparison === object
+          end
+
+        [comparison, object, pass]
+      end
+    end
+  end
+end

data/lib/que/utils/constantization.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Que
+  module Utils
+    module Constantization
+      def constantize(string)
+        Que.assert String, string
+
+        if string.respond_to?(:constantize)
+          string.constantize
+        else
+          names = string.split('::')
+          names.reject!(&:empty?)
+          names.inject(Object, &:const_get)
+        end
+      end
+    end
+  end
+end

data/lib/que/utils/error_notification.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Que
+  module Utils
+    module ErrorNotification
+      attr_accessor :error_notifier
+
+      def notify_error(*args)
+        Que.internal_log(:error_notification_attempted) do
+          {args: args.inspect}
+        end
+
+        if notifier = error_notifier
+          arity = notifier.arity
+          args = args.first(arity) if arity >= 0
+
+          notifier.call(*args)
+        end
+      rescue => error
+        Que.log(
+          event:   :error_notifier_failed,
+          level:   :error,
+          message: "error_notifier callable raised an error",
+
+          error_class:     error.class.name,
+          error_message:   error.message,
+          error_backtrace: error.backtrace,
+        )
+        nil
+      end
+
+      ASYNC_QUEUE    = Queue.new
+      MAX_QUEUE_SIZE = 5
+
+      # Helper method to notify errors asynchronously. For use in high-priority
+      # code, where we don't want to be held up by whatever I/O the error
+      # notification proc contains.
+      def notify_error_async(*args)
+        # We don't synchronize around the size check and the push, so there's a
+        # race condition where the queue could grow to more than the maximum
+        # number of errors, but no big deal if it does. The size check is mainly
+        # here to ensure that the error queue doesn't grow unboundedly large in
+        # pathological cases.
+
+        if ASYNC_QUEUE.size < MAX_QUEUE_SIZE
+          ASYNC_QUEUE.push(args)
+          # Puma raises some ugly warnings if you start up a new thread in the
+          # background during initialization, so start the async error-reporting
+          # thread lazily.
+          async_error_thread
+          true
+        else
+          false
+        end
+      end
+
+      def async_error_thread
+        CONFIG_MUTEX.synchronize do
+          @async_error_thread ||=
+            Thread.new do
+              Thread.current.abort_on_exception = true
+              loop { Que.notify_error(*ASYNC_QUEUE.pop) }
+            end
+        end
+      end
+    end
+  end
+end

data/lib/que/utils/freeze.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Helper method for recursively freezing a data structure.
+
+module Que
+  module Utils
+    module Freeze
+      def recursively_freeze(thing)
+        case thing
+        when Array
+          thing.each { |e| recursively_freeze(e) }
+        when Hash
+          thing.each { |k, v| recursively_freeze(k); recursively_freeze(v) }
+        end
+
+        thing.freeze
+      end
+    end
+  end
+end

data/lib/que/utils/introspection.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Tools for introspecting the state of the job queue.
+
+module Que
+  module Utils
+    module Introspection
+      SQL[:job_stats] =
+        %{
+          SELECT job_class,
+                 count(*)                    AS count,
+                 count(locks.id)             AS count_working,
+                 sum((error_count > 0)::int) AS count_errored,
+                 max(error_count)            AS highest_error_count,
+                 min(run_at)                 AS oldest_run_at
+          FROM public.que_jobs
+          LEFT JOIN (
+            SELECT (classid::bigint << 32) + objid::bigint AS id
+            FROM pg_locks
+            WHERE locktype = 'advisory'
+          ) locks USING (id)
+          WHERE finished_at IS NULL AND expired_at IS NULL
+          GROUP BY job_class
+          ORDER BY count(*) DESC
+        }
+
+      def job_stats
+        execute :job_stats
+      end
+
+      SQL[:job_states] =
+        %{
+          SELECT que_jobs.*,
+                 pg.ruby_hostname,
+                 pg.ruby_pid
+          FROM public.que_jobs
+          JOIN (
+            SELECT (classid::bigint << 32) + objid::bigint AS id, que_lockers.*
+            FROM pg_locks
+            JOIN public.que_lockers USING (pid)
+            WHERE locktype = 'advisory'
+          ) pg USING (id)
+        }
+
+      def job_states
+        execute :job_states
+      end
+    end
+  end
+end

data/lib/que/utils/json_serialization.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Logic for serializing to/from JSON. We assume that the standard library's JSON
+# module is good enough for our purposes.
+
+require 'json'
+
+module Que
+  module Utils
+    module JSONSerialization
+      def serialize_json(object)
+        JSON.dump(object)
+      end
+
+      def deserialize_json(json)
+        # Allowing `create_additions` would be a security vulnerability.
+        JSON.parse(json, symbolize_names: true, create_additions: false)
+      end
+    end
+  end
+end

data/lib/que/utils/logging.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# Tools for logging from Que.
+
+module Que
+  module Utils
+    module Logging
+      attr_accessor :logger, :internal_logger
+      attr_writer :log_formatter
+
+      def log(event:, level: :info, **extra)
+        data = _default_log_data
+        data[:event] = Que.assert(Symbol, event)
+        data.merge!(extra)
+
+        if l = get_logger
+          begin
+            if output = log_formatter.call(data)
+              l.send level, output
+            end
+          rescue => e
+            msg =
+              "Error raised from Que.log_formatter proc:" +
+              " #{e.class}: #{e.message}\n#{e.backtrace}"
+
+            l.error(msg)
+          end
+        end
+      end
+
+      # Logging method used specifically to instrument Que's internals. There's
+      # usually not an internal logger set up, so this method is generally a no-
+      # op unless the specs are running or someone turns on internal logging so
+      # we can debug an issue.
+      def internal_log(event, object = nil)
+        if l = get_logger(internal: true)
+          data = _default_log_data
+
+          data[:internal_event] = Que.assert(Symbol, event)
+          data[:object_id]      = object.object_id if object
+          data[:t]              = Time.now.utc.iso8601(6)
+
+          additional = Que.assert(Hash, yield)
+
+          # Make sure that none of our log contents accidentally overwrite our
+          # default data contents.
+          expected_length = data.length + additional.length
+          data.merge!(additional)
+          Que.assert(expected_length == data.length) do
+            "Bad internal logging keys in: #{additional.keys.inspect}"
+          end
+
+          l.info(JSON.dump(data))
+        end
+      end
+
+      def get_logger(internal: false)
+        if l = internal ? internal_logger : logger
+          l.respond_to?(:call) ? l.call : l
+        end
+      end
+
+      def log_formatter
+        @log_formatter ||= JSON.method(:dump)
+      end
+
+      private
+
+      def _default_log_data
+        {
+          lib:      :que,
+          hostname: CURRENT_HOSTNAME,
+          pid:      Process.pid,
+          thread:   Thread.current.object_id,
+        }
+      end
+    end
+  end
+end