que 0.14.3 → 1.0.0.beta

data/lib/que/rails/railtie.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Que
+  module Rails
+    class Railtie < Rails::Railtie
+      config.que = Que
+
+      Que.run_asynchronously = true if Rails.env.test?
+
+      Que.logger     = proc { Rails.logger }
+      Que.connection = ::ActiveRecord if defined? ::ActiveRecord
+    end
+  end
+end

data/lib/que/result_queue.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# A thread-safe queue that holds ids for jobs that have been worked. Allows
+# appending single/retrieving all ids in a thread-safe fashion.
+
+module Que
+  class ResultQueue
+    def initialize
+      @array = []
+      @mutex = Mutex.new
+    end
+
+    def push(item)
+      sync { @array.push(item) }
+    end
+
+    def clear
+      sync { @array.pop(@array.size) }
+    end
+
+    def to_a
+      sync { @array.dup }
+    end
+
+    def length
+      sync { @array.length }
+    end
+
+    private
+
+    def sync
+      @mutex.synchronize { yield }
+    end
+  end
+end

data/lib/que/sequel/model.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Que
+  module Sequel
+    QUALIFIED_TABLE = ::Sequel.qualify(:public, :que_jobs)
+
+    class Model < ::Sequel::Model(QUALIFIED_TABLE)
+      dataset_module do
+        conditions = {
+          errored:   ::Sequel.qualify(QUALIFIED_TABLE, :error_count) > 0,
+          expired:   ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :expired_at)  => nil),
+          finished:  ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :finished_at) => nil),
+          scheduled: ::Sequel.qualify(QUALIFIED_TABLE, :run_at) > ::Sequel::CURRENT_TIMESTAMP,
+        }
+
+        conditions.each do |name, condition|
+          subset           name,  condition
+          subset :"not_#{name}", ~condition
+        end
+
+        subset :ready,     conditions.values.map(&:~).inject{|a, b| a & b}
+        subset :not_ready, conditions.values.         inject{|a, b| a | b}
+
+        def by_job_class(job_class)
+          job_class = job_class.name if job_class.is_a?(Class)
+          where(
+            ::Sequel.|(
+              {::Sequel.qualify(QUALIFIED_TABLE, :job_class) => job_class},
+              {
+                ::Sequel.qualify(QUALIFIED_TABLE, :job_class) => "ActiveJob::QueueAdapters::QueAdapter::JobWrapper",
+                ::Sequel.lit("public.que_jobs.args->0->>'job_class'") => job_class,
+              }
+            )
+          )
+        end
+
+        def by_queue(queue)
+          where(::Sequel.qualify(QUALIFIED_TABLE, :queue) => queue)
+        end
+
+        def by_tag(tag)
+          where(::Sequel.lit("public.que_jobs.data @> ?", JSON.dump(tags: [tag])))
+        end
+
+        def by_args(*args)
+          where(::Sequel.lit("public.que_jobs.args @> ?", JSON.dump(args)))
+        end
+      end
+    end
+  end
+end

data/lib/que/utils/assertions.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# Assertion helpers. Que has a fair amount of internal state, and there's no
+# telling what users will try to throw at it, so for ease of debugging issues it
+# makes sense to sanity-check frequently.
+
+module Que
+  module Utils
+    module Assertions
+      class AssertionFailed < Error; end
+
+      def assert(*args)
+        comparison, object, pass = _check_assertion_args(*args)
+        return object if pass
+
+        message =
+          if block_given?
+            yield.to_s
+          elsif comparison
+            "Expected #{comparison.inspect}, got #{object.inspect}!"
+          else
+            "Assertion failed!"
+          end
+
+        # Remove this method from the backtrace, to make errors clearer.
+        raise AssertionFailed, message, caller
+      end
+
+      def assert?(*args)
+        _, _, pass = _check_assertion_args(*args)
+        !!pass
+      end
+
+      private
+
+      # Want to support:
+      #   assert(x)                       # Truthiness.
+      #   assert(thing, other)            # Trip-equals.
+      #   assert([thing1, thing2], other) # Multiple Trip-equals.
+      def _check_assertion_args(first, second = (second_omitted = true; nil))
+        if second_omitted
+          comparison = nil
+          object     = first
+        else
+          comparison = first
+          object     = second
+        end
+
+        pass =
+          if second_omitted
+            object
+          elsif comparison.is_a?(Array)
+            comparison.any? { |k| k === object }
+          else
+            comparison === object
+          end
+
+        [comparison, object, pass]
+      end
+    end
+  end
+end

data/lib/que/utils/constantization.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Que
+  module Utils
+    module Constantization
+      def constantize(string)
+        assert String, string
+
+        if string.respond_to?(:constantize)
+          string.constantize
+        else
+          names = string.split('::')
+          names.reject!(&:empty?)
+          names.inject(Object, &:const_get)
+        end
+      end
+    end
+  end
+end

data/lib/que/utils/error_notification.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Que
+  module Utils
+    module ErrorNotification
+      attr_accessor :error_notifier
+
+      def notify_error(*args)
+        Que.internal_log(:error_notification_attempted) do
+          {args: args.inspect}
+        end
+
+        if notifier = error_notifier
+          arity = notifier.arity
+          args = args.first(arity) if arity >= 0
+
+          notifier.call(*args)
+        end
+      rescue => error
+        Que.log(
+          event:   :error_notifier_failed,
+          level:   :error,
+          message: "error_notifier callable raised an error",
+
+          error_class:     error.class.name,
+          error_message:   error.message,
+          error_backtrace: error.backtrace,
+        )
+        nil
+      end
+
+      ASYNC_QUEUE    = Queue.new
+      MAX_QUEUE_SIZE = 5
+
+      # Helper method to notify errors asynchronously. For use in high-priority
+      # code, where we don't want to be held up by whatever I/O the error
+      # notification proc contains.
+      def notify_error_async(*args)
+        # We don't synchronize around the size check and the push, so there's a
+        # race condition where the queue could grow to more than the maximum
+        # number of errors, but no big deal if it does. The size check is mainly
+        # here to ensure that the error queue doesn't grow unboundedly large in
+        # pathological cases.
+
+        if ASYNC_QUEUE.size < MAX_QUEUE_SIZE
+          ASYNC_QUEUE.push(args)
+          # Puma raises some ugly warnings if you start up a new thread in the
+          # background during initialization, so start the async error-reporting
+          # thread lazily.
+          async_error_thread
+          true
+        else
+          false
+        end
+      end
+
+      def async_error_thread
+        CONFIG_MUTEX.synchronize do
+          @async_error_thread ||=
+            Thread.new do
+              Thread.current.abort_on_exception = true
+              loop { Que.notify_error(*ASYNC_QUEUE.pop) }
+            end
+        end
+      end
+    end
+  end
+end

data/lib/que/utils/freeze.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Helper method for recursively freezing a data structure.
+
+module Que
+  module Utils
+    module Freeze
+      def recursively_freeze(thing)
+        case thing
+        when Array
+          thing.each { |e| recursively_freeze(e) }
+        when Hash
+          thing.each { |k, v| recursively_freeze(k); recursively_freeze(v) }
+        end
+
+        thing.freeze
+      end
+    end
+  end
+end

data/lib/que/utils/introspection.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# Tools for introspecting the state of the job queue.
+
+module Que
+  module Utils
+    module Introspection
+      SQL[:job_stats] =
+        %{
+          SELECT job_class,
+                 count(*)                    AS count,
+                 count(locks.id)             AS count_working,
+                 sum((error_count > 0)::int) AS count_errored,
+                 max(error_count)            AS highest_error_count,
+                 min(run_at)                 AS oldest_run_at
+          FROM public.que_jobs
+          LEFT JOIN (
+            SELECT (classid::bigint << 32) + objid::bigint AS id
+            FROM pg_locks
+            WHERE locktype = 'advisory'
+          ) locks USING (id)
+          WHERE finished_at IS NULL AND expired_at IS NULL
+          GROUP BY job_class
+          ORDER BY count(*) DESC
+        }
+
+      def job_stats
+        execute :job_stats
+      end
+
+      SQL[:job_states] =
+        %{
+          SELECT que_jobs.*,
+                 pg.ruby_hostname,
+                 pg.ruby_pid
+          FROM public.que_jobs
+          JOIN (
+            SELECT (classid::bigint << 32) + objid::bigint AS id, que_lockers.*
+            FROM pg_locks
+            JOIN public.que_lockers USING (pid)
+            WHERE locktype = 'advisory'
+          ) pg USING (id)
+        }
+
+      def job_states
+        execute :job_states
+      end
+    end
+  end
+end

data/lib/que/utils/json_serialization.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Logic for serializing to/from JSON. We assume that the standard library's JSON
+# module is good enough for our purposes.
+
+require 'json'
+
+module Que
+  module Utils
+    module JSONSerialization
+      def serialize_json(object)
+        JSON.dump(object)
+      end
+
+      def deserialize_json(json)
+        # Allowing `create_additions` would be a security vulnerability.
+        JSON.parse(json, symbolize_names: true, create_additions: false)
+      end
+    end
+  end
+end

data/lib/que/utils/logging.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# Tools for logging from Que.
+
+module Que
+  module Utils
+    module Logging
+      attr_accessor :logger, :log_formatter, :internal_logger
+
+      def log(event:, level: :info, **extra)
+        data = _default_log_data
+        data[:event] = Que.assert(Symbol, event)
+        data.merge!(extra)
+
+        if l = get_logger
+          begin
+            if output = log_formatter.call(data)
+              l.send level, output
+            end
+          rescue => e
+            msg =
+              "Error raised from Que.log_formatter proc:" +
+              " #{e.class}: #{e.message}\n#{e.backtrace}"
+
+            l.error(msg)
+          end
+        end
+      end
+
+      # Logging method used specifically to instrument Que's internals. There's
+      # usually not an internal logger set up, so this method is generally a no-
+      # op unless the specs are running or someone turns on internal logging so
+      # we can debug an issue.
+      def internal_log(event, object = nil)
+        if l = get_logger(internal: true)
+          data = _default_log_data
+
+          data[:internal_event] = Que.assert(Symbol, event)
+          data[:object_id]      = object.object_id if object
+          data[:t]              = Time.now.utc.iso8601(6)
+
+          additional = Que.assert(Hash, yield)
+
+          # Make sure that none of our log contents accidentally overwrite our
+          # default data contents.
+          expected_length = data.length + additional.length
+          data.merge!(additional)
+          Que.assert(expected_length == data.length) do
+            "Bad internal logging keys in: #{additional.keys.inspect}"
+          end
+
+          l.info(JSON.dump(data))
+        end
+      end
+
+      def get_logger(internal: false)
+        if l = internal ? internal_logger : logger
+          l.respond_to?(:call) ? l.call : l
+        end
+      end
+
+      def log_formatter
+        @log_formatter ||= JSON.method(:dump)
+      end
+
+      private
+
+      def _default_log_data
+        {
+          lib:      :que,
+          hostname: CURRENT_HOSTNAME,
+          pid:      Process.pid,
+          thread:   Thread.current.object_id,
+        }
+      end
+    end
+  end
+end

data/lib/que/utils/middleware.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# Logic for middleware to wrap jobs.
+
+module Que
+  module Utils
+    module Middleware
+      def run_middleware(job, &block)
+        invoke_middleware(
+          middleware: middleware.dup,
+          job:        job,
+          block:      block,
+        )
+      end
+
+      def middleware
+        @middleware ||= []
+      end
+
+      private
+
+      def invoke_middleware(middleware:, job:, block:)
+        if m = middleware.shift
+          m.call(job) do
+            invoke_middleware(middleware: middleware, job: job, block: block)
+          end
+        else
+          block.call
+        end
+      end
+    end
+  end
+end