que 0.11.3 → 2.2.0

data/lib/que/active_job/extensions.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Que
+  module ActiveJob
+    # A module that devs can include into their ApplicationJob classes to get
+    # access to Que-like job behavior.
+    module JobExtensions
+      include JobMethods
+
+      def run(*args)
+        raise Error, "Job class #{self.class} didn't define a run() method!"
+      end
+
+      def perform(*args)
+        args, kwargs = Que.split_out_ruby2_keywords(args)
+
+        Que.internal_log(:active_job_perform, self) do
+          {args: args, kwargs: kwargs}
+        end
+
+        _run(
+          args: Que.recursively_freeze(
+            que_filter_args(
+              args.map { |a| a.is_a?(Hash) ? a.deep_symbolize_keys : a }
+            )
+          ),
+          kwargs: Que.recursively_freeze(
+            que_filter_args(
+              kwargs.deep_symbolize_keys,
+            )
+          ),
+        )
+      end
+
+      private
+
+      # Have helper methods like `destroy` and `retry_in` delegate to the actual
+      # job object. If the current job is being run through an ActiveJob adapter
+      # other than Que's, this will return nil, which is fine.
+      def que_target
+        Thread.current[:que_current_job]
+      end
+
+      # Filter out :_aj_symbol_keys constructs so that keywords work as
+      # expected.
+      def que_filter_args(thing)
+        case thing
+        when Array
+          thing.map { |t| que_filter_args(t) }
+        when Hash
+          thing.each_with_object({}) do |(k, v), hash|
+            hash[k] = que_filter_args(v) unless k == :_aj_symbol_keys
+          end
+        else
+          thing
+        end
+      end
+    end
+
+    # A module that we mix into ActiveJob's wrapper for Que::Job, to maintain
+    # backwards-compatibility with internal changes we make.
+    module WrapperExtensions
+      module ClassMethods
+        # We've dropped support for job options supplied as top-level keywords, but ActiveJob's QueAdapter still uses them. So we have to move them into the job_options hash ourselves.
+        def enqueue(args, priority:, queue:, run_at: nil)
+          super(args, job_options: { priority: priority, queue: queue, run_at: run_at })
+        end
+      end
+
+      module InstanceMethods
+        # The Rails adapter (built against a pre-1.0 version of this gem)
+        # assumes that it can access a job's id via job.attrs["job_id"]. So,
+        # oblige it.
+        def attrs
+          {"job_id" => que_attrs[:id]}
+        end
+
+        def run(args)
+          # Our ActiveJob extensions expect to be able to operate on the actual
+          # job object, but there's no way to access it through ActiveJob. So,
+          # scope it to the current thread. It's a bit messy, but it's the best
+          # option under the circumstances (doesn't require hacking ActiveJob in
+          # any more extensive way).
+
+          # There's no reason this logic should ever nest, because it wouldn't
+          # make sense to run a worker inside of a job, but even so, assert that
+          # nothing absurd is going on.
+          Que.assert NilClass, Thread.current[:que_current_job]
+
+          begin
+            Thread.current[:que_current_job] = self
+
+            # We symbolize the args hash but ActiveJob doesn't like that :/
+            super(args.deep_stringify_keys)
+          ensure
+            # Also assert that the current job state was only removed now, but
+            # unset the job first so that an assertion failure doesn't mess up
+            # the state any more than it already has.
+            current = Thread.current[:que_current_job]
+            Thread.current[:que_current_job] = nil
+            Que.assert(self, current)
+          end
+        end
+      end
+    end
+  end
+end
+
+class ActiveJob::QueueAdapters::QueAdapter
+  class JobWrapper < Que::Job
+    extend Que::ActiveJob::WrapperExtensions::ClassMethods
+    prepend Que::ActiveJob::WrapperExtensions::InstanceMethods
+  end
+end

data/lib/que/active_record/connection.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Que
+  module ActiveRecord
+    module Connection
+      class << self
+        private
+
+        # Check out a PG::Connection object from ActiveRecord's pool.
+        def checkout
+          wrap_in_rails_executor do
+            ::ActiveRecord::Base.connection_pool.with_connection do |conn|
+               yield conn.raw_connection
+            end
+          end
+        end
+
+        # Use Rails' executor (if present) to make sure that the connection
+        # we're using isn't taken from us while the block runs. See
+        # https://github.com/que-rb/que/issues/166#issuecomment-274218910
+        def wrap_in_rails_executor(&block)
+          if defined?(::Rails.application.executor)
+            ::Rails.application.executor.wrap(&block)
+          else
+            yield
+          end
+        end
+      end
+
+      module JobMiddleware
+        class << self
+          def call(job)
+            yield
+
+            # ActiveRecord will check out connections to the current thread when
+            # queries are executed and not return them to the pool until
+            # explicitly requested to. I'm not wild about this API design, and
+            # it doesn't pose a problem for the typical case of workers using a
+            # single PG connection (since we ensure that connection is checked
+            # in and checked out responsibly), but since ActiveRecord supports
+            # connections to multiple databases, it's easy for people using that
+            # feature to unknowingly leak connections to other databases. So,
+            # take the additional step of telling ActiveRecord to check in all
+            # of the current thread's connections after each job is run.
+            ::ActiveRecord::Base.clear_active_connections!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/que/active_record/model.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Que
+  module ActiveRecord
+    class Model < ::ActiveRecord::Base
+      self.table_name = 'public.que_jobs'
+
+      t = arel_table
+
+      scope :errored,     -> { where(t[:error_count].gt(0)) }
+      scope :not_errored, -> { where(t[:error_count].eq(0)) }
+
+      scope :expired,     -> { where(t[:expired_at].not_eq(nil)) }
+      scope :not_expired, -> { where(t[:expired_at].eq(nil)) }
+
+      scope :finished,     -> { where(t[:finished_at].not_eq(nil)) }
+      scope :not_finished, -> { where(t[:finished_at].eq(nil)) }
+
+      scope :scheduled,     -> { where(t[:run_at].gt  (Arel.sql("now()"))) }
+      scope :not_scheduled, -> { where(t[:run_at].lteq(Arel.sql("now()"))) }
+
+      scope :ready,     -> { not_errored.not_expired.not_finished.not_scheduled }
+      scope :not_ready, -> { where(t[:error_count].gt(0).or(t[:expired_at].not_eq(nil)).or(t[:finished_at].not_eq(nil)).or(t[:run_at].gt(Arel.sql("now()")))) }
+
+      class << self
+        def by_job_class(job_class)
+          job_class = job_class.name if job_class.is_a?(Class)
+          where(
+            "que_jobs.job_class = ? OR (que_jobs.job_class = 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper' AND que_jobs.args->0->>'job_class' = ?)",
+            job_class, job_class,
+          )
+        end
+
+        def by_queue(queue)
+          where(arel_table[:queue].eq(queue))
+        end
+
+        def by_tag(tag)
+          where("que_jobs.data @> ?", JSON.dump(tags: [tag]))
+        end
+
+        def by_args(*args, **kwargs)
+          where("que_jobs.args @> ? AND que_jobs.kwargs @> ?", JSON.dump(args), JSON.dump(kwargs))
+        end
+      end
+    end
+  end
+end

data/lib/que/command_line_interface.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+require 'optparse'
+require 'logger'
+require 'uri'
+
+module Que
+  module CommandLineInterface
+    # Have a sensible default require file for Rails.
+    RAILS_ENVIRONMENT_FILE = './config/environment.rb'
+
+    class << self
+      # Need to rely on dependency injection a bit to make this method cleanly
+      # testable :/
+      def parse(
+        args:,
+        output:,
+        default_require_file: RAILS_ENVIRONMENT_FILE
+      )
+
+        options           = {}
+        queues            = []
+        log_level         = 'info'
+        log_internals     = false
+        poll_interval     = 5
+        connection_url    = nil
+        worker_count      = nil
+        worker_priorities = nil
+
+        parser =
+          OptionParser.new do |opts|
+            opts.banner = 'usage: que [options] [file/to/require] ...'
+
+            opts.on(
+              '-h',
+              '--help',
+              "Show this help text.",
+            ) do
+              output.puts opts.help
+              return 0
+            end
+
+            opts.on(
+              '-i',
+              '--poll-interval [INTERVAL]',
+              Float,
+              "Set maximum interval between polls for available jobs, " \
+                "in seconds (default: 5)",
+            ) do |i|
+              poll_interval = i
+            end
+
+            opts.on(
+              '-l',
+              '--log-level [LEVEL]',
+              String,
+              "Set level at which to log to STDOUT " \
+                "(debug, info, warn, error, fatal) (default: info)",
+            ) do |l|
+              log_level = l
+            end
+
+            opts.on(
+              '-p',
+              '--worker-priorities [LIST]',
+              Array,
+              "List of priorities to assign to workers (default: 10,30,50,any,any,any)",
+            ) do |priority_array|
+              worker_priorities =
+                priority_array.map do |p|
+                  case p
+                  when /\Aany\z/i
+                    nil
+                  when /\A\d+\z/
+                    Integer(p)
+                  else
+                    output.puts "Invalid priority option: '#{p}'. Please use an integer or the word 'any'."
+                    return 1
+                  end
+                end
+            end
+
+            opts.on(
+              '-q',
+              '--queue-name [NAME]',
+              String,
+              "Set a queue name to work jobs from. " \
+                "Can be passed multiple times. " \
+                "(default: the default queue only)",
+            ) do |queue_name|
+              queues << queue_name
+            end
+
+            opts.on(
+              '-w',
+              '--worker-count [COUNT]',
+              Integer,
+              "Set number of workers in process (default: 6)",
+            ) do |w|
+              worker_count = w
+            end
+
+            opts.on(
+              '-v',
+              '--version',
+              "Print Que version and exit.",
+            ) do
+              require 'que'
+              output.puts "Que version #{Que::VERSION}"
+              return 0
+            end
+
+            opts.on(
+              '--connection-url [URL]',
+              String,
+              "Set a custom database url to connect to for locking purposes.",
+            ) do |url|
+              options[:connection_url] = url
+            end
+
+            opts.on(
+              '--log-internals',
+              "Log verbosely about Que's internal state. " \
+                "Only recommended for debugging issues",
+            ) do |l|
+              log_internals = true
+            end
+
+            opts.on(
+              '--maximum-buffer-size [SIZE]',
+              Integer,
+              "Set maximum number of jobs to be locked and held in this " \
+                "process awaiting a worker (default: 8)",
+            ) do |s|
+              options[:maximum_buffer_size] = s
+            end
+
+            opts.on(
+              '--minimum-buffer-size [SIZE]',
+              Integer,
+              "Unused (deprecated)",
+            ) do |s|
+              warn "The --minimum-buffer-size SIZE option has been deprecated and will be removed in v2.0 (it's actually been unused since v1.0.0.beta4)"
+            end
+
+            opts.on(
+              '--wait-period [PERIOD]',
+              Float,
+              "Set maximum interval between checks of the in-memory job queue, " \
+                "in milliseconds (default: 50)",
+            ) do |p|
+              options[:wait_period] = p
+            end
+          end
+
+        parser.parse!(args)
+
+        options[:worker_priorities] =
+          if worker_count && worker_priorities
+            worker_priorities.values_at(0...worker_count)
+          elsif worker_priorities
+            worker_priorities
+          elsif worker_count
+            Array.new(worker_count) { nil }
+          else
+            [10, 30, 50, nil, nil, nil]
+          end
+
+        if args.length.zero?
+          if File.exist?(default_require_file)
+            args << default_require_file
+          else
+            output.puts <<-OUTPUT
+You didn't include any Ruby files to require!
+Que needs to be able to load your application before it can process jobs.
+(Or use `que -h` for a list of options)
+OUTPUT
+            return 1
+          end
+        end
+
+        args.each do |file|
+          begin
+            require file
+          rescue LoadError => e
+            output.puts "Could not load file '#{file}': #{e}"
+            return 1
+          end
+        end
+
+        Que.logger ||= Logger.new(STDOUT)
+
+        if log_internals
+          Que.internal_logger = Que.logger
+        end
+
+        begin
+          Que.get_logger.level = Logger.const_get(log_level.upcase)
+        rescue NameError
+          output.puts "Unsupported logging level: #{log_level} (try debug, info, warn, error, or fatal)"
+          return 1
+        end
+
+        if queues.any?
+          queues_hash = {}
+
+          queues.each do |queue|
+            name, interval = queue.split('=')
+            p              = interval ? Float(interval) : poll_interval
+
+            Que.assert(p > 0.01) { "Poll intervals can't be less than 0.01 seconds!" }
+
+            queues_hash[name] = p
+          end
+
+          options[:queues] = queues_hash
+        end
+
+        options[:poll_interval] = poll_interval
+
+        locker =
+          begin
+            Que::Locker.new(**options)
+          rescue => e
+            output.puts(e.message)
+            return 1
+          end
+
+        # It's a bit sloppy to use a global for this when a local variable would
+        # do, but we want to stop the locker from the CLI specs, so...
+        $stop_que_executable = false
+        %w[INT TERM].each { |signal| trap(signal) { $stop_que_executable = true } }
+
+        output.puts(
+          <<~STARTUP
+            Que #{Que::VERSION} started worker process with:
+              Worker threads: #{locker.workers.length} (priorities: #{locker.workers.map { |w| w.priority || 'any' }.join(', ')})
+              Buffer size: #{locker.job_buffer.maximum_size}
+              Queues:
+            #{locker.queues.map { |queue, interval| "    - #{queue} (poll interval: #{interval}s)" }.join("\n")}
+            Que waiting for jobs...
+          STARTUP
+        )
+
+        loop do
+          sleep 0.01
+          break if $stop_que_executable || locker.stopping?
+        end
+
+        output.puts "\nFinishing Que's current jobs before exiting..."
+
+        locker.stop!
+
+        output.puts "Que's jobs finished, exiting..."
+        return 0
+      end
+    end
+  end
+end

data/lib/que/connection.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+# A simple wrapper class around connections that basically just improves the
+# query API a bit. Currently, our connection pool wrapper discards these
+# connection wrappers once the connection is returned to the source connection
+# pool, so this class isn't currently suitable for storing data about the
+# connection long-term (like what statements it has prepared, for example).
+
+# If we wanted to do that, we'd probably need to sneak a reference to the
+# wrapper into the PG::Connection object itself, by just setting a instance
+# variable that's something namespaced and hopefully safe, like
+# `@que_connection_wrapper`. It's a bit ugly, but it should ensure that we don't
+# cause any memory leaks in esoteric setups where one-off connections are being
+# established and then garbage-collected.
+
+require 'time' # For Time.parse
+require 'set'
+
+module Que
+  class Connection
+    extend Forwardable
+
+    attr_reader :wrapped_connection
+
+    def_delegators :wrapped_connection, :backend_pid, :wait_for_notify
+
+    class << self
+      def wrap(conn)
+        case conn
+        when self
+          conn
+        when PG::Connection
+          if conn.instance_variable_defined?(:@que_wrapper)
+            conn.instance_variable_get(:@que_wrapper)
+          else
+            conn.instance_variable_set(:@que_wrapper, new(conn))
+          end
+        else
+          raise Error, "Unsupported input for Connection.wrap: #{conn.class}"
+        end
+      end
+    end
+
+    def initialize(connection)
+      @wrapped_connection = connection
+      @prepared_statements = Set.new
+    end
+
+    def execute(command, params = [])
+      sql =
+        case command
+        when Symbol then SQL[command]
+        when String then command
+        else raise Error, "Bad command! #{command.inspect}"
+        end
+
+      params = convert_params(params)
+
+      result =
+        Que.run_sql_middleware(sql, params) do
+          # Some versions of the PG gem dislike an empty/nil params argument.
+          if params.empty?
+            wrapped_connection.async_exec(sql)
+          else
+            wrapped_connection.async_exec_params(sql, params)
+          end
+        end
+
+      Que.internal_log :connection_execute, self do
+        {
+          backend_pid: backend_pid,
+          command:     command,
+          params:      params,
+          ntuples:     result.ntuples,
+        }
+      end
+
+      convert_result(result)
+    end
+
+    def execute_prepared(command, params = nil)
+      Que.assert(Symbol, command)
+
+      if !Que.use_prepared_statements || in_transaction?
+        return execute(command, params)
+      end
+
+      name = "que_#{command}"
+
+      begin
+        unless @prepared_statements.include?(command)
+          wrapped_connection.prepare(name, SQL[command])
+          @prepared_statements.add(command)
+          prepared_just_now = true
+        end
+
+        convert_result(
+          wrapped_connection.exec_prepared(name, params)
+        )
+      rescue ::PG::InvalidSqlStatementName => error
+        # Reconnections on ActiveRecord can cause the same connection
+        # objects to refer to new backends, so recover as well as we can.
+
+        unless prepared_just_now
+          Que.log level: :warn, event: :reprepare_statement, command: command
+          @prepared_statements.delete(command)
+          retry
+        end
+
+        raise error
+      end
+    end
+
+    def next_notification
+      wrapped_connection.notifies
+    end
+
+    def drain_notifications
+      loop { break if next_notification.nil? }
+    end
+
+    def server_version
+      wrapped_connection.server_version
+    end
+
+    def in_transaction?
+      wrapped_connection.transaction_status != ::PG::PQTRANS_IDLE
+    end
+
+    private
+
+    def convert_params(params)
+      params.map do |param|
+        case param
+        when Time
+          # The pg gem unfortunately doesn't convert fractions of time
+          # instances, so cast them to a string.
+          param.strftime('%Y-%m-%d %H:%M:%S.%6N %z')
+        when Array, Hash
+          # Handle JSON.
+          Que.serialize_json(param)
+        else
+          param
+        end
+      end
+    end
+
+    # Procs used to convert strings from Postgres into Ruby types.
+    CAST_PROCS = {
+      # Boolean
+      16 => -> (value) {
+        case value
+        when String then value == 't'.freeze
+        else !!value
+        end
+      },
+
+      # Timestamp with time zone
+      1184 => -> (value) {
+        case value
+        when Time then value
+        when String then Time.parse(value)
+        else raise "Unexpected time class: #{value.class} (#{value.inspect})"
+        end
+      }
+    }
+
+    # JSON, JSONB
+    CAST_PROCS[114] = CAST_PROCS[3802] = -> (j) { Que.deserialize_json(j) }
+
+    # Integer, bigint, smallint
+    CAST_PROCS[23] = CAST_PROCS[20] = CAST_PROCS[21] = proc(&:to_i)
+
+    CAST_PROCS.freeze
+
+    def convert_result(result)
+      output = result.to_a
+
+      result.fields.each_with_index do |field, index|
+        symbol = field.to_sym
+
+        if converter = CAST_PROCS[result.ftype(index)]
+          output.each do |hash|
+            value = hash.delete(field)
+            value = converter.call(value) if value
+            hash[symbol] = value
+          end
+        else
+          output.each do |hash|
+            hash[symbol] = hash.delete(field)
+          end
+        end
+      end
+
+      output
+    end
+  end
+end