queue_classic 3.2.0.RC1 → 4.0.0.pre.alpha1

data/lib/queue_classic/conn_adapter.rb

@@ -1,27 +1,37 @@
+# frozen_string_literal: true
+
 require 'uri'
 require 'pg'
 
 module QC
   class ConnAdapter
 
-    attr_accessor :connection
-    def initialize(c=nil)
-      @connection = c.nil? ? establish_new : validate!(c)
+    def initialize(args={})
+      @active_record_connection_share = args[:active_record_connection_share]
+      @_connection = args[:connection]
       @mutex = Mutex.new
     end
 
+    def connection
+      if @active_record_connection_share && Object.const_defined?('ActiveRecord')
+        ActiveRecord::Base.connection.raw_connection
+      else
+        @_connection ||= establish_new
+      end
+    end
+
     def execute(stmt, *params)
       @mutex.synchronize do
         QC.log(:at => "exec_sql", :sql => stmt.inspect)
         begin
           params = nil if params.empty?
-          r = @connection.exec(stmt, params)
+          r = connection.exec(stmt, params)
           result = []
           r.each {|t| result << t}
           result.length > 1 ? result : result.pop
-        rescue PGError => e
+        rescue PG::Error => e
           QC.log(:error => e.inspect)
-          @connection.reset
+          connection.reset
           raise
         end
       end
@@ -30,10 +40,10 @@ module QC
     def wait(time, *channels)
       @mutex.synchronize do
         listen_cmds = channels.map {|c| 'LISTEN "' + c.to_s + '"'}
-        @connection.exec(listen_cmds.join(';'))
+        connection.exec(listen_cmds.join(';'))
         wait_for_notify(time)
         unlisten_cmds = channels.map {|c| 'UNLISTEN "' + c.to_s + '"'}
-        @connection.exec(unlisten_cmds.join(';'))
+        connection.exec(unlisten_cmds.join(';'))
         drain_notify
       end
     end
@@ -41,7 +51,7 @@ module QC
     def disconnect
       @mutex.synchronize do
         begin
-          @connection.close
+          connection.close
         rescue => e
           QC.log(:at => 'disconnect', :error => e.message)
         end
@@ -59,12 +69,12 @@ module QC
 
     def wait_for_notify(t)
       Array.new.tap do |msgs|
-        @connection.wait_for_notify(t) {|event, pid, msg| msgs << msg}
+        connection.wait_for_notify(t) {|event, pid, msg| msgs << msg}
       end
     end
 
     def drain_notify
-      until @connection.notifies.nil?
+      until connection.notifies.nil?
         QC.log(:at => "drain_notifications")
       end
     end
@@ -77,10 +87,15 @@ module QC
 
     def establish_new
       QC.log(:at => "establish_conn")
-      conn = PGconn.connect(*normalize_db_url(db_url))
-      if conn.status != PGconn::CONNECTION_OK
+      conn = PG.connect(*normalize_db_url(db_url))
+      if conn.status != PG::CONNECTION_OK
         QC.log(:error => conn.error)
       end
+
+      if conn.server_version < 90600
+        raise "This version of Queue Classic does not support Postgres older than 9.6 (90600). This version is #{conn.server_version}. If you need that support, please use an older version."
+      end
+
       conn.exec("SET application_name = '#{QC.app_name}'")
       conn
     end
@@ -106,6 +121,5 @@ module QC
             raise(ArgumentError, "missing QC_DATABASE_URL or DATABASE_URL")
       @db_url = URI.parse(url)
     end
-
   end
 end

data/lib/queue_classic/queue.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'conn_adapter'
 require 'json'
 require 'time'
@@ -5,8 +7,8 @@ require 'time'
 module QC
   # The queue class maps a queue abstraction onto a database table.
   class Queue
-
     attr_reader :name, :top_bound
+
     def initialize(name, top_bound=nil)
       @name = name
       @top_bound = top_bound || QC.top_bound
@@ -38,7 +40,16 @@ module QC
     def enqueue(method, *args)
       QC.log_yield(:measure => 'queue.enqueue') do
         s = "INSERT INTO #{QC.table_name} (q_name, method, args) VALUES ($1, $2, $3) RETURNING id"
-        conn_adapter.execute(s, name, method, JSON.dump(args))
+        begin
+          retries ||= 0
+          conn_adapter.execute(s, name, method, JSON.dump(args))
+        rescue PG::Error
+          if (retries += 1) < 2
+            retry
+          else
+            raise
+          end
+        end
       end
     end
 
@@ -64,14 +75,42 @@ module QC
         s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at)
              VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')
              RETURNING id"
-        conn_adapter.execute(s, name, method, JSON.dump(args))
+        begin
+          retries ||= 0
+          conn_adapter.execute(s, name, method, JSON.dump(args))
+        rescue PG::Error
+          if (retries += 1) < 2
+            retry
+          else
+            raise
+          end
+        end
       end
     end
 
     def lock
       QC.log_yield(:measure => 'queue.lock') do
-        s = "SELECT * FROM lock_head($1, $2)"
-        if r = conn_adapter.execute(s, name, top_bound)
+        s = <<~SQL
+          WITH selected_job AS (
+            SELECT id
+            FROM queue_classic_jobs
+            WHERE
+              locked_at IS NULL AND
+              q_name = $1 AND
+              scheduled_at <= now()
+            LIMIT 1
+            FOR NO KEY UPDATE SKIP LOCKED
+          )
+          UPDATE queue_classic_jobs
+          SET
+            locked_at = now(),
+            locked_by = pg_backend_pid()
+          FROM selected_job
+          WHERE queue_classic_jobs.id = selected_job.id
+          RETURNING *
+        SQL
+
+        if r = conn_adapter.execute(s, name)
           {}.tap do |job|
             job[:id] = r["id"]
             job[:q_name] = r["q_name"]
@@ -89,14 +128,14 @@ module QC
 
     def unlock(id)
       QC.log_yield(:measure => 'queue.unlock') do
-        s = "UPDATE #{QC.table_name} set locked_at = null where id = $1"
+        s = "UPDATE #{QC.table_name} SET locked_at = NULL WHERE id = $1"
         conn_adapter.execute(s, id)
       end
     end
 
     def delete(id)
       QC.log_yield(:measure => 'queue.delete') do
-        conn_adapter.execute("DELETE FROM #{QC.table_name} where id = $1", id)
+        conn_adapter.execute("DELETE FROM #{QC.table_name} WHERE id = $1", id)
       end
     end
 
@@ -107,13 +146,28 @@ module QC
       end
     end
 
+    # Count the number of jobs in a specific queue. This returns all
+    # jobs, including ones that are scheduled in the future.
     def count
-      QC.log_yield(:measure => 'queue.count') do
-        s = "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1"
-        r = conn_adapter.execute(s, name)
+      _count('queue.count', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1")
+    end
+
+    # Count the number of jobs in a specific queue, except ones scheduled in the future
+    def count_ready
+      _count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at <= now()")
+    end
+
+    # Count the number of jobs in a specific queue scheduled in the future
+    def count_scheduled
+      _count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at > now()")
+    end
+
+    private
+    def _count(metric_name, sql)
+      QC.log_yield(measure: metric_name) do
+        r = conn_adapter.execute(sql, name)
         r["count"].to_i
       end
     end
-
   end
 end

data/lib/queue_classic/railtie.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'rails/railtie'
 
 module QC

data/lib/queue_classic/setup.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module QC
   module Setup
     Root = File.expand_path("../..", File.dirname(__FILE__))
@@ -8,51 +10,66 @@ module QC
     DowngradeFrom_3_0_0 = File.join(Root, "/sql/downgrade_from_3_0_0.sql")
     UpgradeTo_3_1_0 = File.join(Root, "/sql/update_to_3_1_0.sql")
     DowngradeFrom_3_1_0 = File.join(Root, "/sql/downgrade_from_3_1_0.sql")
+    UpgradeTo_4_0_0 = File.join(Root, "/sql/update_to_4_0_0.sql")
+    DowngradeFrom_4_0_0 = File.join(Root, "/sql/downgrade_from_4_0_0.sql")
 
     def self.create(c = QC::default_conn_adapter.connection)
-      conn = QC::ConnAdapter.new(c)
+      conn = QC::ConnAdapter.new(connection: c)
       conn.execute(File.read(CreateTable))
       conn.execute(File.read(SqlFunctions))
       conn.disconnect if c.nil? #Don't close a conn we didn't create.
     end
 
     def self.drop(c = QC::default_conn_adapter.connection)
-      conn = QC::ConnAdapter.new(c)
+      conn = QC::ConnAdapter.new(connection: c)
       conn.execute("DROP TABLE IF EXISTS queue_classic_jobs CASCADE")
       conn.execute(File.read(DropSqlFunctions))
       conn.disconnect if c.nil? #Don't close a conn we didn't create.
     end
 
     def self.update(c = QC::default_conn_adapter.connection)
-      conn = QC::ConnAdapter.new(c)
+      conn = QC::ConnAdapter.new(connection: c)
       conn.execute(File.read(UpgradeTo_3_0_0))
       conn.execute(File.read(UpgradeTo_3_1_0))
+      conn.execute(File.read(UpgradeTo_4_0_0))
       conn.execute(File.read(DropSqlFunctions))
       conn.execute(File.read(SqlFunctions))
     end
 
     def self.update_to_3_0_0(c = QC::default_conn_adapter.connection)
-      conn = QC::ConnAdapter.new(c)
+      conn = QC::ConnAdapter.new(connection: c)
       conn.execute(File.read(UpgradeTo_3_0_0))
       conn.execute(File.read(DropSqlFunctions))
       conn.execute(File.read(SqlFunctions))
     end
 
     def self.downgrade_from_3_0_0(c = QC::default_conn_adapter.connection)
-      conn = QC::ConnAdapter.new(c)
+      conn = QC::ConnAdapter.new(connection: c)
       conn.execute(File.read(DowngradeFrom_3_0_0))
     end
 
     def self.update_to_3_1_0(c = QC::default_conn_adapter.connection)
-      conn = QC::ConnAdapter.new(c)
+      conn = QC::ConnAdapter.new(connection: c)
       conn.execute(File.read(UpgradeTo_3_1_0))
       conn.execute(File.read(DropSqlFunctions))
       conn.execute(File.read(SqlFunctions))
     end
 
     def self.downgrade_from_3_1_0(c = QC::default_conn_adapter.connection)
-      conn = QC::ConnAdapter.new(c)
+      conn = QC::ConnAdapter.new(connection: c)
       conn.execute(File.read(DowngradeFrom_3_1_0))
     end
+
+    def self.update_to_4_0_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(connection: c)
+      conn.execute(File.read(UpgradeTo_4_0_0))
+      conn.execute(File.read(DropSqlFunctions))
+      conn.execute(File.read(SqlFunctions))
+    end
+
+    def self.downgrade_from_4_0_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(connection: c)
+      conn.execute(File.read(DowngradeFrom_4_0_0))
+    end
   end
 end

data/lib/queue_classic/tasks.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 task :environment
 
 namespace :jobs do
@@ -11,11 +13,8 @@ namespace :qc do
     @worker = QC.default_worker_class.new
 
     trap('INT') do
-      $stderr.puts("Received INT. Shutting down.")
-      if !@worker.running
-        $stderr.puts("Worker has stopped running. Exit.")
-        exit(1)
-      end
+      $stderr.puts("Received INT. Shutting down.")      
+      abort("Worker has stopped running. Exit.") unless @worker.running
       @worker.stop
     end
 

data/lib/queue_classic/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module QC
-  VERSION = "3.2.0.RC1"
+  VERSION = "4.0.0-alpha1"
 end

data/lib/queue_classic/worker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # -*- coding: utf-8 -*-
 require_relative 'queue'
 require_relative 'conn_adapter'
@@ -12,7 +14,7 @@ module QC
     # This method takes a single hash argument. The following keys are read:
     # fork_worker:: Worker forks each job execution.
     # wait_interval:: Time to wait between failed lock attempts
-    # connection:: PGConn object.
+    # connection:: PG::Connection object.
     # q_name:: Name of a single queue to process.
     # q_names:: Names of queues to process. Will process left to right.
     # top_bound:: Offset to the head of the queue. 1 == strict FIFO.
@@ -21,7 +23,7 @@ module QC
       @wait_interval = args[:wait_interval] || QC.wait_time
 
       if args[:connection]
-        @conn_adapter = ConnAdapter.new(args[:connection])
+        @conn_adapter = ConnAdapter.new(connection: args[:connection])
       else
         @conn_adapter = QC.default_conn_adapter
       end
@@ -112,7 +114,10 @@ module QC
           queue.delete(job[:id])
           finished = true
         end
-      rescue => e
+      rescue StandardError, ScriptError, NoMemoryError => e
+        # We really only want to unlock the job for signal and system exit
+        # exceptions. If we encounter a ScriptError or a NoMemoryError any
+        # future run will likely encounter the same error.
         handle_failure(job, e)
         finished = true
       ensure
@@ -134,8 +139,8 @@ module QC
       receiver.send(message, *args)
     end
 
-    # This method will be called when an exception
-    # is raised during the execution of the job.
+    # This method will be called when a StandardError, ScriptError or
+    # NoMemoryError is raised during the execution of the job.
     def handle_failure(job,e)
       $stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect} at=#{e.backtrace.first}")
     end

data/queue_classic.gemspec

@@ -20,5 +20,5 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = %w[lib]
 
-  spec.add_dependency "pg", ">= 0.17", "< 0.19"
+  spec.add_dependency "pg", ">= 0.17", "< 2.0"
 end

data/sql/create_table.sql

@@ -1,26 +1,17 @@
-do $$ begin
+DO $$ BEGIN
 
 CREATE TABLE queue_classic_jobs (
   id bigserial PRIMARY KEY,
-  q_name text not null check (length(q_name) > 0),
-  method text not null check (length(method) > 0),
-  args   text not null,
+  q_name text NOT NULL CHECK (length(q_name) > 0),
+  method text NOT NULL CHECK (length(method) > 0),
+  args   jsonb NOT NULL,
   locked_at timestamptz,
   locked_by integer,
-  created_at timestamptz default now(),
-  scheduled_at timestamptz default now()
+  created_at timestamptz DEFAULT now(),
+  scheduled_at timestamptz DEFAULT now()
 );
 
--- If jsonb type is available, use it for the args column
-if exists (select 1 from pg_type where typname = 'jsonb') then
-  alter table queue_classic_jobs alter column args type jsonb using args::jsonb;
--- Otherwise, use json type for the args column if available
-elsif exists (select 1 from pg_type where typname = 'json') then
-  alter table queue_classic_jobs alter column args type json using args::json;
-end if;
-
-end $$ language plpgsql;
+END $$ LANGUAGE plpgsql;
 
 CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
 CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
-

data/sql/ddl.sql

@@ -1,84 +1,8 @@
--- We are declaring the return type to be queue_classic_jobs.
--- This is ok since I am assuming that all of the users added queues will
--- have identical columns to queue_classic_jobs.
--- When QC supports queues with columns other than the default, we will have to change this.
-
-CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
-RETURNS SETOF queue_classic_jobs AS $$
-DECLARE
-  unlocked bigint;
-  relative_top integer;
-  job_count integer;
-BEGIN
-  -- The purpose is to release contention for the first spot in the table.
-  -- The select count(*) is going to slow down dequeue performance but allow
-  -- for more workers. Would love to see some optimization here...
-
-  EXECUTE 'SELECT count(*) FROM '
-    || '(SELECT * FROM queue_classic_jobs '
-    || ' WHERE locked_at IS NULL'
-    || ' AND q_name = '
-    || quote_literal(q_name)
-    || ' AND scheduled_at <= '
-    || quote_literal(now())
-    || ' LIMIT '
-    || quote_literal(top_boundary)
-    || ') limited'
-  INTO job_count;
-
-  SELECT TRUNC(random() * (top_boundary - 1))
-  INTO relative_top;
-
-  IF job_count < top_boundary THEN
-    relative_top = 0;
-  END IF;
-
-  LOOP
-    BEGIN
-      EXECUTE 'SELECT id FROM queue_classic_jobs '
-        || ' WHERE locked_at IS NULL'
-        || ' AND q_name = '
-        || quote_literal(q_name)
-        || ' AND scheduled_at <= '
-        || quote_literal(now())
-        || ' ORDER BY id ASC'
-        || ' LIMIT 1'
-        || ' OFFSET ' || quote_literal(relative_top)
-        || ' FOR UPDATE NOWAIT'
-      INTO unlocked;
-      EXIT;
-    EXCEPTION
-      WHEN lock_not_available THEN
-        -- do nothing. loop again and hope we get a lock
-    END;
-  END LOOP;
-
-  RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
-    || ' SET locked_at = (CURRENT_TIMESTAMP),'
-    || ' locked_by = (select pg_backend_pid())'
-    || ' WHERE id = $1'
-    || ' AND locked_at is NULL'
-    || ' RETURNING *'
-  USING unlocked;
-
-  RETURN;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE FUNCTION lock_head(tname varchar)
-RETURNS SETOF queue_classic_jobs AS $$
-BEGIN
-  RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
-END;
-$$ LANGUAGE plpgsql;
-
 -- queue_classic_notify function and trigger
-create function queue_classic_notify() returns trigger as $$ begin
-  perform pg_notify(new.q_name, '');
-  return null;
-end $$ language plpgsql;
+CREATE FUNCTION queue_classic_notify() RETURNS TRIGGER AS $$ BEGIN
+  perform pg_notify(new.q_name, ''); RETURN NULL;
+END $$ LANGUAGE plpgsql;
 
-create trigger queue_classic_notify
-after insert on queue_classic_jobs
-for each row
-execute procedure queue_classic_notify();
+CREATE TRIGGER queue_classic_notify
+AFTER INSERT ON queue_classic_jobs FOR EACH ROW
+EXECUTE PROCEDURE queue_classic_notify();