que 1.0.0.beta4 → 1.4.0

data/lib/que/job.rb

@@ -12,7 +12,7 @@ module Que
     SQL[:insert_job] =
       %{
         INSERT INTO public.que_jobs
-        (queue, priority, run_at, job_class, args, data)
+        (queue, priority, run_at, job_class, args, data, job_schema_version)
         VALUES
         (
           coalesce($1, 'default')::text,
@@ -20,7 +20,8 @@ module Que
           coalesce($3, now())::timestamptz,
           $4::text,
           coalesce($5, '[]')::jsonb,
-          coalesce($6, '{}')::jsonb
+          coalesce($6, '{}')::jsonb,
+          #{Que.job_schema_version}
         )
         RETURNING *
       }
@@ -57,22 +58,18 @@ module Que
 
       def enqueue(
         *args,
-        queue:     nil,
-        priority:  nil,
-        run_at:    nil,
-        job_class: nil,
-        tags:      nil,
+        job_options: {},
         **arg_opts
       )
-
+        arg_opts, job_options = _extract_job_options(arg_opts, job_options.dup)
         args << arg_opts if arg_opts.any?
 
-        if tags
-          if tags.length > MAXIMUM_TAGS_COUNT
-            raise Que::Error, "Can't enqueue a job with more than #{MAXIMUM_TAGS_COUNT} tags! (passed #{tags.length})"
+        if job_options[:tags]
+          if job_options[:tags].length > MAXIMUM_TAGS_COUNT
+            raise Que::Error, "Can't enqueue a job with more than #{MAXIMUM_TAGS_COUNT} tags! (passed #{job_options[:tags].length})"
           end
 
-          tags.each do |tag|
+          job_options[:tags].each do |tag|
             if tag.length > MAXIMUM_TAG_LENGTH
               raise Que::Error, "Can't enqueue a job with a tag longer than 100 characters! (\"#{tag}\")"
             end
@@ -80,13 +77,13 @@ module Que
         end
 
         attrs = {
-          queue:    queue    || resolve_que_setting(:queue) || Que.default_queue,
-          priority: priority || resolve_que_setting(:priority),
-          run_at:   run_at   || resolve_que_setting(:run_at),
+          queue:    job_options[:queue]    || resolve_que_setting(:queue) || Que.default_queue,
+          priority: job_options[:priority] || resolve_que_setting(:priority),
+          run_at:   job_options[:run_at]   || resolve_que_setting(:run_at),
           args:     Que.serialize_json(args),
-          data:     tags ? Que.serialize_json(tags: tags) : "{}",
+          data:     job_options[:tags] ? Que.serialize_json(tags: job_options[:tags]) : "{}",
           job_class: \
-            job_class || name ||
+            job_options[:job_class] || name ||
               raise(Error, "Can't enqueue an anonymous subclass of Que::Job"),
         }
 
@@ -139,6 +136,27 @@ module Que
           end
         end
       end
+
+      def _extract_job_options(arg_opts, job_options)
+        deprecated_job_option_names = []
+
+        %i[queue priority run_at job_class tags].each do |option_name|
+          next unless arg_opts.key?(option_name) && job_options[option_name].nil?
+
+          job_options[option_name] = arg_opts.delete(option_name)
+          deprecated_job_option_names << option_name
+        end
+
+        _log_job_options_deprecation(deprecated_job_option_names)
+
+        [arg_opts, job_options]
+      end
+
+      def _log_job_options_deprecation(deprecated_job_option_names)
+        return unless deprecated_job_option_names.any?
+
+        warn "Passing job options like (#{deprecated_job_option_names.join(', ')}) to `JobClass.enqueue` as top level keyword args has been deprecated and will be removed in version 2.0. Please wrap job options in an explicit `job_options` keyword arg instead."
+      end
     end
 
     # Set up some defaults.

data/lib/que/job_buffer.rb

@@ -6,7 +6,7 @@
 
 module Que
   class JobBuffer
-    attr_reader :maximum_size, :minimum_size, :priority_queues
+    attr_reader :maximum_size, :priority_queues
 
     # Since we use a mutex, which is not reentrant, we have to be a little
     # careful to not call a method that locks the mutex when we've already
@@ -17,20 +17,11 @@ module Que
 
     def initialize(
       maximum_size:,
-      minimum_size:,
       priorities:
     )
       @maximum_size = Que.assert(Integer, maximum_size)
       Que.assert(maximum_size >= 0) { "maximum_size for a JobBuffer must be at least zero!" }
 
-      @minimum_size = Que.assert(Integer, minimum_size)
-      Que.assert(minimum_size >= 0) { "minimum_size for a JobBuffer must be at least zero!" }
-
-      Que.assert(minimum_size <= maximum_size) do
-        "minimum buffer size (#{minimum_size}) is " \
-          "greater than the maximum buffer size (#{maximum_size})!"
-      end
-
       @stop  = false
       @array = []
       @mutex = Mutex.new
@@ -59,10 +50,8 @@ module Que
 
         # Relying on the hash's contents being sorted, here.
         priority_queues.reverse_each do |_, pq|
-          pq.waiting_count.times do
-            job = _shift_job(pq.priority)
-            break if job.nil? # False would mean we're stopping.
-            pq.push(job)
+          pq.populate do
+            _shift_job(pq.priority)
           end
         end
 
@@ -75,7 +64,7 @@ module Que
 
     def shift(priority = nil)
       queue = priority_queues.fetch(priority) { raise Error, "not a permitted priority! #{priority}" }
-      queue.pop
+      queue.pop || shift_job(priority)
     end
 
     def shift_job(priority = nil)
@@ -158,6 +147,10 @@ module Que
       sync { _stopping? }
     end
 
+    def job_available?(priority)
+      (job = @array.first) && job.priority_sufficient?(priority)
+    end
+
     private
 
     def _buffer_space
@@ -210,15 +203,14 @@ module Que
       def pop
         sync do
           loop do
-            return false if @stopping
-
-            if item = @items.pop
+            if @stopping
+              return false
+            elsif item = @items.pop
               return item
+            elsif job_buffer.job_available?(priority)
+              return false
             end
 
-            job = job_buffer.shift_job(priority)
-            return job unless job.nil? # False means we're stopping.
-
             @waiting += 1
             @cv.wait(mutex)
             @waiting -= 1
@@ -226,18 +218,20 @@ module Que
         end
       end
 
-      def push(item)
+      def stop
         sync do
-          Que.assert(waiting_count > 0)
-          @items << item
-          @cv.signal
+          @stopping = true
+          @cv.broadcast
         end
       end
 
-      def stop
+      def populate
         sync do
-          @stopping = true
-          @cv.broadcast
+          waiting_count.times do
+            job = yield
+            break if job.nil? # False would mean we're stopping.
+            _push(job)
+          end
         end
       end
 
@@ -250,6 +244,12 @@ module Que
       def sync(&block)
         mutex.synchronize(&block)
       end
+
+      def _push(item)
+        Que.assert(waiting_count > 0)
+        @items << item
+        @cv.signal
+      end
     end
   end
 end

data/lib/que/locker.rb

@@ -24,12 +24,12 @@ module Que
 
   SQL[:register_locker] =
     %{
-      INSERT INTO public.que_lockers (pid, worker_count, worker_priorities, ruby_pid, ruby_hostname, listening, queues)
-      VALUES (pg_backend_pid(), $1::integer, $2::integer[], $3::integer, $4::text, $5::boolean, $6::text[])
+      INSERT INTO public.que_lockers (pid, worker_count, worker_priorities, ruby_pid, ruby_hostname, listening, queues, job_schema_version)
+      VALUES (pg_backend_pid(), $1::integer, $2::integer[], $3::integer, $4::text, $5::boolean, $6::text[], $7::integer)
     }
 
   class Locker
-    attr_reader :thread, :workers, :job_buffer, :locks
+    attr_reader :thread, :workers, :job_buffer, :locks, :queues, :poll_interval
 
     MESSAGE_RESOLVERS = {}
     RESULT_RESOLVERS  = {}
@@ -45,7 +45,6 @@ module Que
 
     DEFAULT_POLL_INTERVAL       = 5.0
     DEFAULT_WAIT_PERIOD         = 50
-    DEFAULT_MINIMUM_BUFFER_SIZE = 2
     DEFAULT_MAXIMUM_BUFFER_SIZE = 8
     DEFAULT_WORKER_PRIORITIES   = [10, 30, 50, nil, nil, nil].freeze
 
@@ -57,7 +56,6 @@ module Que
       poll_interval:       DEFAULT_POLL_INTERVAL,
       wait_period:         DEFAULT_WAIT_PERIOD,
       maximum_buffer_size: DEFAULT_MAXIMUM_BUFFER_SIZE,
-      minimum_buffer_size: DEFAULT_MINIMUM_BUFFER_SIZE,
       worker_priorities:   DEFAULT_WORKER_PRIORITIES,
       on_worker_start:     nil
     )
@@ -77,7 +75,6 @@ module Que
       # ResultQueue to receive messages from workers.
       @job_buffer = JobBuffer.new(
         maximum_size: maximum_buffer_size,
-        minimum_size: minimum_buffer_size,
         priorities:   worker_priorities.uniq,
       )
 
@@ -93,7 +90,6 @@ module Que
           poll_interval:       poll_interval,
           wait_period:         wait_period,
           maximum_buffer_size: maximum_buffer_size,
-          minimum_buffer_size: minimum_buffer_size,
           worker_priorities:   worker_priorities,
         }
       end
@@ -101,7 +97,20 @@ module Que
       # Local cache of which advisory locks are held by this connection.
       @locks = Set.new
 
-      @queue_names = queues.is_a?(Hash) ? queues.keys : queues
+      @poll_interval = poll_interval
+
+      if queues.is_a?(Hash)
+        @queue_names = queues.keys
+        @queues = queues.transform_values do |interval|
+          interval || poll_interval
+        end
+      else
+        @queue_names = queues
+        @queues = queues.map do |queue_name|
+          [queue_name, poll_interval]
+        end.to_h
+      end
+
       @wait_period = wait_period.to_f / 1000 # Milliseconds to seconds.
 
       @workers =
@@ -183,11 +192,11 @@ module Que
 
             @pollers =
               if poll
-                queues.map do |queue, interval|
+                @queues.map do |queue_name, interval|
                   Poller.new(
                     connection:    @connection,
-                    queue:         queue,
-                    poll_interval: interval || poll_interval,
+                    queue:         queue_name,
+                    poll_interval: interval,
                   )
                 end
               end
@@ -266,6 +275,7 @@ module Que
         CURRENT_HOSTNAME,
         !!@listener,
         "{\"#{@queue_names.join('","')}\"}",
+        Que.job_schema_version,
       ]
     end
 
@@ -393,10 +403,12 @@ module Que
         }
       end
 
+      materalize_cte = connection.server_version >= 12_00_00
+
       jobs =
         connection.execute \
           <<-SQL
-            WITH jobs AS (SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')}))
+            WITH jobs AS #{materalize_cte ? 'MATERIALIZED' : ''} (SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')}))
             SELECT * FROM jobs WHERE pg_try_advisory_lock(id)
           SQL
 

data/lib/que/migrations/4/up.sql

@@ -146,7 +146,9 @@ CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
         FROM (
           SELECT *
           FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
-          WHERE listening AND queues @> ARRAY[NEW.queue]
+          WHERE
+            listening AND
+            queues @> ARRAY[NEW.queue]
           ORDER BY md5(pid::text || id::text)
         ) t1
       ) t2

data/lib/que/migrations/5/down.sql

@@ -0,0 +1,73 @@
+DROP TRIGGER que_job_notify ON que_jobs;
+DROP FUNCTION que_job_notify();
+
+DROP INDEX que_poll_idx_with_job_schema_version;
+
+ALTER TABLE que_jobs
+  DROP COLUMN job_schema_version;
+
+ALTER TABLE que_lockers
+  DROP COLUMN job_schema_version;
+
+CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
+  DECLARE
+    locker_pid integer;
+    sort_key json;
+  BEGIN
+    -- Don't do anything if the job is scheduled for a future time.
+    IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN
+      RETURN null;
+    END IF;
+
+    -- Pick a locker to notify of the job's insertion, weighted by their number
+    -- of workers. Should bounce pseudorandomly between lockers on each
+    -- invocation, hence the md5-ordering, but still touch each one equally,
+    -- hence the modulo using the job_id.
+    SELECT pid
+    INTO locker_pid
+    FROM (
+      SELECT *, last_value(row_number) OVER () + 1 AS count
+      FROM (
+        SELECT *, row_number() OVER () - 1 AS row_number
+        FROM (
+          SELECT *
+          FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
+          WHERE
+            listening AND
+            queues @> ARRAY[NEW.queue]
+          ORDER BY md5(pid::text || id::text)
+        ) t1
+      ) t2
+    ) t3
+    WHERE NEW.id % count = row_number;
+
+    IF locker_pid IS NOT NULL THEN
+      -- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so
+      -- rather than throw errors when someone enqueues a big job, just
+      -- broadcast the most pertinent information, and let the locker query for
+      -- the record after it's taken the lock. The worker will have to hit the
+      -- DB in order to make sure the job is still visible anyway.
+      SELECT row_to_json(t)
+      INTO sort_key
+      FROM (
+        SELECT
+          'job_available' AS message_type,
+          NEW.queue       AS queue,
+          NEW.priority    AS priority,
+          NEW.id          AS id,
+          -- Make sure we output timestamps as UTC ISO 8601
+          to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at
+      ) t;
+
+      PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text);
+    END IF;
+
+    RETURN null;
+  END
+$$
+LANGUAGE plpgsql;
+
+CREATE TRIGGER que_job_notify
+  AFTER INSERT ON que_jobs
+  FOR EACH ROW
+  EXECUTE PROCEDURE public.que_job_notify();

data/lib/que/migrations/5/up.sql

@@ -0,0 +1,76 @@
+DROP TRIGGER que_job_notify ON que_jobs;
+DROP FUNCTION que_job_notify();
+
+ALTER TABLE que_jobs
+  ADD COLUMN job_schema_version INTEGER DEFAULT 1;
+
+ALTER TABLE que_lockers
+  ADD COLUMN job_schema_version INTEGER DEFAULT 1;
+
+CREATE INDEX que_poll_idx_with_job_schema_version
+  ON que_jobs (job_schema_version, queue, priority, run_at, id)
+  WHERE (finished_at IS NULL AND expired_at IS NULL);
+
+CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
+  DECLARE
+    locker_pid integer;
+    sort_key json;
+  BEGIN
+    -- Don't do anything if the job is scheduled for a future time.
+    IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN
+      RETURN null;
+    END IF;
+
+    -- Pick a locker to notify of the job's insertion, weighted by their number
+    -- of workers. Should bounce pseudorandomly between lockers on each
+    -- invocation, hence the md5-ordering, but still touch each one equally,
+    -- hence the modulo using the job_id.
+    SELECT pid
+    INTO locker_pid
+    FROM (
+      SELECT *, last_value(row_number) OVER () + 1 AS count
+      FROM (
+        SELECT *, row_number() OVER () - 1 AS row_number
+        FROM (
+          SELECT *
+          FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
+          WHERE
+            listening AND
+            queues @> ARRAY[NEW.queue] AND
+            ql.job_schema_version = NEW.job_schema_version
+          ORDER BY md5(pid::text || id::text)
+        ) t1
+      ) t2
+    ) t3
+    WHERE NEW.id % count = row_number;
+
+    IF locker_pid IS NOT NULL THEN
+      -- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so
+      -- rather than throw errors when someone enqueues a big job, just
+      -- broadcast the most pertinent information, and let the locker query for
+      -- the record after it's taken the lock. The worker will have to hit the
+      -- DB in order to make sure the job is still visible anyway.
+      SELECT row_to_json(t)
+      INTO sort_key
+      FROM (
+        SELECT
+          'job_available' AS message_type,
+          NEW.queue       AS queue,
+          NEW.priority    AS priority,
+          NEW.id          AS id,
+          -- Make sure we output timestamps as UTC ISO 8601
+          to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at
+      ) t;
+
+      PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text);
+    END IF;
+
+    RETURN null;
+  END
+$$
+LANGUAGE plpgsql;
+
+CREATE TRIGGER que_job_notify
+  AFTER INSERT ON que_jobs
+  FOR EACH ROW
+  EXECUTE PROCEDURE public.que_job_notify();

data/lib/que/migrations.rb

@@ -4,7 +4,7 @@ module Que
   module Migrations
     # In order to ship a schema change, add the relevant up and down sql files
     # to the migrations directory, and bump the version here.
-    CURRENT_VERSION = 4
+    CURRENT_VERSION = 5
 
     class << self
       def migrate!(version:)
@@ -28,7 +28,6 @@ module Que
               step,
               direction,
             ].join('/') << '.sql'
-
             Que.execute(File.read(filename))
           end
 

data/lib/que/poller.rb

@@ -68,6 +68,7 @@ module Que
             SELECT j
             FROM public.que_jobs AS j
             WHERE queue = $1::text
+              AND job_schema_version = #{Que.job_schema_version}
               AND NOT id = ANY($2::bigint[])
               AND priority <= pg_temp.que_highest_remaining_priority($3::jsonb)
               AND run_at <= now()
@@ -88,6 +89,7 @@ module Que
                   SELECT j
                   FROM public.que_jobs AS j
                   WHERE queue = $1::text
+                    AND job_schema_version = #{Que.job_schema_version}
                     AND NOT id = ANY($2::bigint[])
                     AND priority <= pg_temp.que_highest_remaining_priority(jobs.remaining_priorities)
                     AND run_at <= now()
@@ -144,8 +146,6 @@ module Que
 
       return unless should_poll?
 
-      expected_count = priorities.inject(0){|s,(_,c)| s + c}
-
       jobs =
         connection.execute_prepared(
           :poll_jobs,
@@ -157,7 +157,7 @@ module Que
         )
 
       @last_polled_at      = Time.now
-      @last_poll_satisfied = expected_count == jobs.count
+      @last_poll_satisfied = poll_satisfied?(priorities, jobs)
 
       Que.internal_log :poller_polled, self do
         {
@@ -263,5 +263,12 @@ module Que
         SQL
       end
     end
+
+    private
+
+    def poll_satisfied?(priorities, jobs)
+      lowest_priority = priorities.keys.max
+      jobs.count >= priorities[lowest_priority]
+    end
   end
 end

data/lib/que/version.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
 module Que
-  VERSION = '1.0.0.beta4'
+  VERSION = '1.4.0'
+
+  def self.job_schema_version
+    1
+  end
 end

data/lib/que/worker.rb

@@ -54,10 +54,17 @@ module Que
     private
 
     def work_loop
-      # Blocks until a job of the appropriate priority is available. If the
-      # queue is shutting down this will return nil, which breaks the loop and
+      # Blocks until a job of the appropriate priority is available.
+      # `fetch_next_metajob` normally returns a job to be processed.
+      # If the queue is shutting down it will return false, which breaks the loop and
       # lets the thread finish.
-      while metajob = fetch_next_metajob
+      while (metajob = fetch_next_metajob) != false
+        # If metajob is nil instead of false, we've hit a rare race condition where
+        # there was a job in the buffer when the worker code checked, but the job was
+        # picked up by the time we got around to shifting it off the buffer.
+        # Letting this case go unhandled leads to worker threads exiting pre-maturely, so
+        # we check explicitly and continue the loop.
+        next if metajob.nil?
         id = metajob.id
 
         Que.internal_log(:worker_received_job, self) { {id: id} }
@@ -130,6 +137,7 @@ module Que
         error: {
           class:   error.class.to_s,
           message: error.message,
+          backtrace: (error.backtrace || []).join("\n").slice(0, 10000),
         },
       )
 
@@ -157,7 +165,7 @@ module Que
           Que.execute :set_error, [
             delay,
             "#{error.class}: #{error.message}".slice(0, 500),
-            error.backtrace.join("\n").slice(0, 10000),
+            (error.backtrace || []).join("\n").slice(0, 10000),
             job.fetch(:id),
           ]
         end

data/lib/que.rb

@@ -31,6 +31,8 @@ module Que
   require_relative 'que/utils/queue_management'
   require_relative 'que/utils/transactions'
 
+  require_relative 'que/version'
+
   require_relative 'que/connection'
   require_relative 'que/connection_pool'
   require_relative 'que/job_methods'
@@ -41,7 +43,6 @@ module Que
   require_relative 'que/migrations'
   require_relative 'que/poller'
   require_relative 'que/result_queue'
-  require_relative 'que/version'
   require_relative 'que/worker'
 
   class << self

data/scripts/docker-entrypoint

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+# For using your own dotfiles within the Docker container
+if [ -f /.docker-rc.d/.docker-bashrc ]; then
+  echo "source /.docker-rc.d/.docker-bashrc" >> ~/.bashrc
+fi
+
+gem list -i -e bundler -v "$RUBY_BUNDLER_VERSION" >/dev/null || gem install bundler -v "$RUBY_BUNDLER_VERSION"
+
+bundle check --dry-run || bundle install
+
+exec "${@-bash}"

data/scripts/test

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+bundle exec rake spec "$@"