que 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 967eded27a81df945b65911275dcdc7925113727bfd06fbc42cb780a3bfb6fcf
-  data.tar.gz: 87607b262263b13623b4b09822a478c02179e57cfa503f253e3e678ac8b1490a
+  metadata.gz: 382e70b455239bf58eda4d83e5772400b55055f98857d2b83fdfc7f36f26d856
+  data.tar.gz: dfad699d5bb996c965e3216de224fc9248a3cb2be70cce04f8407ed9b5ee0651
 SHA512:
-  metadata.gz: 3d8242fe07445a6f4658f322dd1a7cdb8918c8a56c2bfad2db49c58695b4ebae183e6a56159abc4ff74316006e7d95459ab6cfd8a4c98410eea50a2e4fb52509
-  data.tar.gz: 24fdf63cab7ff6e8a2ffa6ed598718d3c2ae2cbc6ef61de22ab6f1541b0cc23bab980fe5d10bbcb54823b2a529cd1cdc9d57437db3905e802bec92a04ea3ed94
+  metadata.gz: d1c34fc4b2ff8ce02b9b70f8fe9e9d361c8e4e701e36987690464a5f62263905e29453c9dbcb65bce9bda19d511d1430b1e235f1ad8e86cbd1e136e189e71015
+  data.tar.gz: 9cb2182726f813fb53c3cb6f2a41dc4bb482810448673a8f8e704860db119e1ad44a64613f3d688df1085f0686ff51ea74a0d1b46bc7353d79d81945cf3555e8

data/CHANGELOG.md

@@ -1,3 +1,36 @@
+### 1.3.0 (2022-02-25)
+
+**ACTION REQUIRED**
+
+This release will allow you to safely upgrade to Que 2 when it comes out, without first needing to empty your `que_jobs` table.
+
+**You will need to first update to this version, apply the Que schema migration, and deploy, before you can safely begin the process of upgrading to Que 2.**
+
+Que 2 will bring Ruby 3 support, but to do that, the job arguments in the `que_jobs` table will need to be split into two columns - repurposing the existing one for positional arguments only (`args`), and adding a new one for keyword arguments (`kwargs`). This is so that Que running in Ruby 3, when reading job arguments stored in the database, can disambiguate between keyword arguments and a last positional argument hash.
+
+The args split hasn't happened yet, but when it does, we still need to be able to successfully process all the existing queued jobs which have their keyword arguments in the `args` column still. Our solution is for you to have both Que 1 workers and Que 2 workers operating simultaneously during the upgrade, each processing only the jobs enqueued from that version. Once all the Que 1 jobs are processed, the Que 1 workers can be retired.
+
+To allow the different worker versions to tell which jobs belong to which, we've added a new column to the `que_jobs` table in this version, `job_schema_version`. Jobs enqueued with Que 1 will have a `1` here, and jobs from Que 2 will have a `2`. Que schema migration 5 will default the job schema version of all existing jobs to `1`.
+
+You will need to migrate Que to the latest Que schema version (5). For instance, on ActiveRecord and Rails 6:
+
+```ruby
+class UpdateQueTablesToVersion5 < ActiveRecord::Migration[6.0]
+  def up
+    Que.migrate!(version: 5)
+  end
+  def down
+    Que.migrate!(version: 4)
+  end
+end
+```
+
+You must apply the schema migration and deploy to upgrade all workers.
+
+No further action is required from you at this stage. The Que 2 release changelog will provide full upgrade instructions for the process briefly described above of simultaneously running both Que 1 & 2 workers. Note that you won't be required to upgrade from Ruby 2.7 to Ruby 3 at the same time as upgrading to Que 2.
+
+If you use any Que plugins or custom code that interacts with the `que_jobs` table, before you can upgrade to Que 2, you will need to make sure they are updated for compatibility with Que 2: They'll need to make use of the `kwargs` column, and when inserting jobs, put the result of `Que.job_schema_version` into the `job_schema_version` column rather than continue to rely on its default of `1`.
+
 ### 1.2.0 (2022-02-23)
 
 - **Deprecated**

data/README.md

@@ -59,7 +59,7 @@ class CreateQueSchema < ActiveRecord::Migration[5.0]
     # Whenever you use Que in a migration, always specify the version you're
     # migrating to. If you're unsure what the current version is, check the
     # changelog.
-    Que.migrate!(version: 4)
+    Que.migrate!(version: 5)
   end
 
   def down
@@ -253,3 +253,29 @@ If you want to try a different version of Postgres, e.g. 12:
 ```bash
 export POSTGRES_VERSION=12
 ```
+
+### Git pre-push hook
+
+So we can avoid breaking the build, we've created Git pre-push hooks to verify everything is ok before pushing.
+
+To set up the pre-push hook locally, run:
+
+```bash
+echo -e "#\!/bin/bash\n\$(dirname \$0)/../../auto/pre-push-hook" > .git/hooks/pre-push
+chmod +x .git/hooks/pre-push
+```
+
+### Release process
+
+The process for releasing a new version of the gem is:
+
+- Merge PR(s)
+- Git pull locally
+- Update the version number, bundle install, and commit
+- Update `CHANGELOG.md`, and commit
+- Tag the commit with the version number, prefixed by `v`
+- Git push to master
+- Git push the tag
+- Publish the new version of the gem to RubyGems: `gem build -o que.gem && gem push que.gem`
+- Create a GitHub release - rather than describe anything there, link to the heading for the release in `CHANGELOG.md`
+- Post on the Que Discord in `#announcements`

data/auto/pre-push-hook

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+cd "$(dirname "$0")/.."
+
+green='\e[32m'; blue='\e[36m'; red='\e[31m'; bold='\e[1m'; reset='\e[0m'
+coloured-arrow() { printf "$bold$1==> $2$reset\n"; }
+success() { coloured-arrow "$green" "$1"; }
+info() { coloured-arrow "$blue" "$1"; }
+err() { coloured-arrow "$red" "$1"; exit 1; }
+
+info 'Running pre-push hook...'
+
+on-exit() {
+  [[ -n "${succeeded-}" ]] || err 'Pre-push checks failed'
+}
+trap on-exit EXIT
+
+info 'Checking for uncommitted changes...'
+[[ -z $(git status -s) ]] || err 'ERROR: You have uncommited changes'
+
+info 'Checking bundle...'
+bundle check --dry-run || bundle install
+
+info 'Specs...'
+auto/test
+
+succeeded=true
+success 'All pre-push checks passed! =)'

data/bin/command_line_interface.rb

@@ -232,7 +232,16 @@ OUTPUT
         $stop_que_executable = false
         %w[INT TERM].each { |signal| trap(signal) { $stop_que_executable = true } }
 
-        output.puts "Que waiting for jobs..."
+        output.puts(
+          <<~STARTUP
+            Que #{Que::VERSION} started worker process with:
+              Worker threads: #{locker.workers.length} (priorities: #{locker.workers.map { |w| w.priority || 'any' }.join(', ')})
+              Buffer size: #{locker.job_buffer.minimum_size}-#{locker.job_buffer.maximum_size}
+              Queues:
+            #{locker.queues.map { |queue, interval| "    - #{queue} (poll interval: #{interval}s)" }.join("\n")}
+            Que waiting for jobs...
+          STARTUP
+        )
 
         loop do
           sleep 0.01

data/docs/README.md

@@ -129,11 +129,11 @@ There are other docs to read if you're using [Sequel](#using-sequel) or [plain P
 After you've connected Que to the database, you can manage the jobs table. You'll want to migrate to a specific version in a migration file, to ensure that they work the same way even when you upgrade Que in the future:
 
 ```ruby
-# Update the schema to version #4.
-Que.migrate! version: 4
+# Update the schema to version #5.
+Que.migrate!(version: 5)
 
 # Remove Que's jobs table entirely.
-Que.migrate! version: 0
+Que.migrate!(version: 0)
 ```
 
 There's also a helper method to clear all jobs from the jobs table:
@@ -405,11 +405,11 @@ Some new releases of Que may require updates to the database schema. It's recomm
 ```ruby
 class UpdateQue < ActiveRecord::Migration[5.0]
   def self.up
-    Que.migrate! version: 3
+    Que.migrate!(version: 3)
   end
 
   def self.down
-    Que.migrate! version: 2
+    Que.migrate!(version: 2)
   end
 end
 ```
@@ -418,7 +418,7 @@ This will make sure that your database schema stays consistent with your codebas
 
 ```ruby
 # Change schema to version 3.
-Que.migrate! version: 3
+Que.migrate!(version: 3)
 
 # Check your current schema version.
 Que.db_version #=> 3
@@ -550,11 +550,11 @@ require 'que'
 Sequel.migration do
   up do
     Que.connection = self
-    Que.migrate! :version => 3
+    Que.migrate!(version: 5)
   end
   down do
     Que.connection = self
-    Que.migrate! :version => 0
+    Que.migrate!(version: 0)
   end
 end
 ```

data/lib/que/job.rb

@@ -12,7 +12,7 @@ module Que
     SQL[:insert_job] =
       %{
         INSERT INTO public.que_jobs
-        (queue, priority, run_at, job_class, args, data)
+        (queue, priority, run_at, job_class, args, data, job_schema_version)
         VALUES
         (
           coalesce($1, 'default')::text,
@@ -20,7 +20,8 @@ module Que
           coalesce($3, now())::timestamptz,
           $4::text,
           coalesce($5, '[]')::jsonb,
-          coalesce($6, '{}')::jsonb
+          coalesce($6, '{}')::jsonb,
+          #{Que.job_schema_version}
         )
         RETURNING *
       }

data/lib/que/locker.rb

@@ -24,12 +24,12 @@ module Que
 
   SQL[:register_locker] =
     %{
-      INSERT INTO public.que_lockers (pid, worker_count, worker_priorities, ruby_pid, ruby_hostname, listening, queues)
-      VALUES (pg_backend_pid(), $1::integer, $2::integer[], $3::integer, $4::text, $5::boolean, $6::text[])
+      INSERT INTO public.que_lockers (pid, worker_count, worker_priorities, ruby_pid, ruby_hostname, listening, queues, job_schema_version)
+      VALUES (pg_backend_pid(), $1::integer, $2::integer[], $3::integer, $4::text, $5::boolean, $6::text[], $7::integer)
     }
 
   class Locker
-    attr_reader :thread, :workers, :job_buffer, :locks
+    attr_reader :thread, :workers, :job_buffer, :locks, :queues, :poll_interval
 
     MESSAGE_RESOLVERS = {}
     RESULT_RESOLVERS  = {}
@@ -101,7 +101,20 @@ module Que
       # Local cache of which advisory locks are held by this connection.
       @locks = Set.new
 
-      @queue_names = queues.is_a?(Hash) ? queues.keys : queues
+      @poll_interval = poll_interval
+
+      if queues.is_a?(Hash)
+        @queue_names = queues.keys
+        @queues = queues.transform_values do |interval|
+          interval || poll_interval
+        end
+      else
+        @queue_names = queues
+        @queues = queues.map do |queue_name|
+          [queue_name, poll_interval]
+        end.to_h
+      end
+
       @wait_period = wait_period.to_f / 1000 # Milliseconds to seconds.
 
       @workers =
@@ -183,11 +196,11 @@ module Que
 
             @pollers =
               if poll
-                queues.map do |queue, interval|
+                @queues.map do |queue_name, interval|
                   Poller.new(
                     connection:    @connection,
-                    queue:         queue,
-                    poll_interval: interval || poll_interval,
+                    queue:         queue_name,
+                    poll_interval: interval,
                   )
                 end
               end
@@ -266,6 +279,7 @@ module Que
         CURRENT_HOSTNAME,
         !!@listener,
         "{\"#{@queue_names.join('","')}\"}",
+        Que.job_schema_version,
       ]
     end
 

data/lib/que/migrations/4/up.sql

@@ -146,7 +146,9 @@ CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
         FROM (
           SELECT *
           FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
-          WHERE listening AND queues @> ARRAY[NEW.queue]
+          WHERE
+            listening AND
+            queues @> ARRAY[NEW.queue]
           ORDER BY md5(pid::text || id::text)
         ) t1
       ) t2

data/lib/que/migrations/5/down.sql

@@ -0,0 +1,73 @@
+DROP TRIGGER que_job_notify ON que_jobs;
+DROP FUNCTION que_job_notify();
+
+DROP INDEX que_poll_idx_with_job_schema_version;
+
+ALTER TABLE que_jobs
+  DROP COLUMN job_schema_version;
+
+ALTER TABLE que_lockers
+  DROP COLUMN job_schema_version;
+
+CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
+  DECLARE
+    locker_pid integer;
+    sort_key json;
+  BEGIN
+    -- Don't do anything if the job is scheduled for a future time.
+    IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN
+      RETURN null;
+    END IF;
+
+    -- Pick a locker to notify of the job's insertion, weighted by their number
+    -- of workers. Should bounce pseudorandomly between lockers on each
+    -- invocation, hence the md5-ordering, but still touch each one equally,
+    -- hence the modulo using the job_id.
+    SELECT pid
+    INTO locker_pid
+    FROM (
+      SELECT *, last_value(row_number) OVER () + 1 AS count
+      FROM (
+        SELECT *, row_number() OVER () - 1 AS row_number
+        FROM (
+          SELECT *
+          FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
+          WHERE
+            listening AND
+            queues @> ARRAY[NEW.queue]
+          ORDER BY md5(pid::text || id::text)
+        ) t1
+      ) t2
+    ) t3
+    WHERE NEW.id % count = row_number;
+
+    IF locker_pid IS NOT NULL THEN
+      -- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so
+      -- rather than throw errors when someone enqueues a big job, just
+      -- broadcast the most pertinent information, and let the locker query for
+      -- the record after it's taken the lock. The worker will have to hit the
+      -- DB in order to make sure the job is still visible anyway.
+      SELECT row_to_json(t)
+      INTO sort_key
+      FROM (
+        SELECT
+          'job_available' AS message_type,
+          NEW.queue       AS queue,
+          NEW.priority    AS priority,
+          NEW.id          AS id,
+          -- Make sure we output timestamps as UTC ISO 8601
+          to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at
+      ) t;
+
+      PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text);
+    END IF;
+
+    RETURN null;
+  END
+$$
+LANGUAGE plpgsql;
+
+CREATE TRIGGER que_job_notify
+  AFTER INSERT ON que_jobs
+  FOR EACH ROW
+  EXECUTE PROCEDURE public.que_job_notify();

data/lib/que/migrations/5/up.sql

@@ -0,0 +1,76 @@
+DROP TRIGGER que_job_notify ON que_jobs;
+DROP FUNCTION que_job_notify();
+
+ALTER TABLE que_jobs
+  ADD COLUMN job_schema_version INTEGER DEFAULT 1;
+
+ALTER TABLE que_lockers
+  ADD COLUMN job_schema_version INTEGER DEFAULT 1;
+
+CREATE INDEX que_poll_idx_with_job_schema_version
+  ON que_jobs (job_schema_version, queue, priority, run_at, id)
+  WHERE (finished_at IS NULL AND expired_at IS NULL);
+
+CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
+  DECLARE
+    locker_pid integer;
+    sort_key json;
+  BEGIN
+    -- Don't do anything if the job is scheduled for a future time.
+    IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN
+      RETURN null;
+    END IF;
+
+    -- Pick a locker to notify of the job's insertion, weighted by their number
+    -- of workers. Should bounce pseudorandomly between lockers on each
+    -- invocation, hence the md5-ordering, but still touch each one equally,
+    -- hence the modulo using the job_id.
+    SELECT pid
+    INTO locker_pid
+    FROM (
+      SELECT *, last_value(row_number) OVER () + 1 AS count
+      FROM (
+        SELECT *, row_number() OVER () - 1 AS row_number
+        FROM (
+          SELECT *
+          FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
+          WHERE
+            listening AND
+            queues @> ARRAY[NEW.queue] AND
+            ql.job_schema_version = NEW.job_schema_version
+          ORDER BY md5(pid::text || id::text)
+        ) t1
+      ) t2
+    ) t3
+    WHERE NEW.id % count = row_number;
+
+    IF locker_pid IS NOT NULL THEN
+      -- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so
+      -- rather than throw errors when someone enqueues a big job, just
+      -- broadcast the most pertinent information, and let the locker query for
+      -- the record after it's taken the lock. The worker will have to hit the
+      -- DB in order to make sure the job is still visible anyway.
+      SELECT row_to_json(t)
+      INTO sort_key
+      FROM (
+        SELECT
+          'job_available' AS message_type,
+          NEW.queue       AS queue,
+          NEW.priority    AS priority,
+          NEW.id          AS id,
+          -- Make sure we output timestamps as UTC ISO 8601
+          to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at
+      ) t;
+
+      PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text);
+    END IF;
+
+    RETURN null;
+  END
+$$
+LANGUAGE plpgsql;
+
+CREATE TRIGGER que_job_notify
+  AFTER INSERT ON que_jobs
+  FOR EACH ROW
+  EXECUTE PROCEDURE public.que_job_notify();

data/lib/que/migrations.rb

@@ -4,7 +4,7 @@ module Que
   module Migrations
     # In order to ship a schema change, add the relevant up and down sql files
     # to the migrations directory, and bump the version here.
-    CURRENT_VERSION = 4
+    CURRENT_VERSION = 5
 
     class << self
       def migrate!(version:)
@@ -28,7 +28,6 @@ module Que
               step,
               direction,
             ].join('/') << '.sql'
-
             Que.execute(File.read(filename))
           end
 

data/lib/que/poller.rb

@@ -68,6 +68,7 @@ module Que
             SELECT j
             FROM public.que_jobs AS j
             WHERE queue = $1::text
+              AND job_schema_version = #{Que.job_schema_version}
               AND NOT id = ANY($2::bigint[])
               AND priority <= pg_temp.que_highest_remaining_priority($3::jsonb)
               AND run_at <= now()
@@ -88,6 +89,7 @@ module Que
                   SELECT j
                   FROM public.que_jobs AS j
                   WHERE queue = $1::text
+                    AND job_schema_version = #{Que.job_schema_version}
                     AND NOT id = ANY($2::bigint[])
                     AND priority <= pg_temp.que_highest_remaining_priority(jobs.remaining_priorities)
                     AND run_at <= now()

data/lib/que/version.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
 module Que
-  VERSION = '1.2.0'
+  VERSION = '1.3.0'
+
+  def self.job_schema_version
+    1
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: que
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Chris Hanks
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-23 00:00:00.000000000 Z
+date: 2022-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -40,6 +40,7 @@ files:
 - README.md
 - Rakefile
 - auto/dev
+- auto/pre-push-hook
 - auto/psql
 - auto/test
 - auto/test-postgres-14
@@ -68,6 +69,8 @@ files:
 - lib/que/migrations/3/up.sql
 - lib/que/migrations/4/down.sql
 - lib/que/migrations/4/up.sql
+- lib/que/migrations/5/down.sql
+- lib/que/migrations/5/up.sql
 - lib/que/poller.rb
 - lib/que/rails/railtie.rb
 - lib/que/result_queue.rb
@@ -106,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.6
 signing_key:
 specification_version: 4
 summary: A PostgreSQL-based Job Queue