queue_classic_plus 4.0.0.alpha17 → 4.0.0.alpha19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b17bbec023aa5f8fa91527f15c91db3ca6a9975da20c304188b5100757bab9b
-  data.tar.gz: 57ce60a1dfd1912b7a6b57c6b8c312f78fc3f903dc674739099dc27bb138d8aa
+  metadata.gz: 4ded39f47849c6dacb9e4efe8bc535bf9fae56cf15cc366cdde177e12176ae37
+  data.tar.gz: 228a9d4ed7b5a4541e1d93e9887c3f3ec9193e9464c9766817ffd1690530d864
 SHA512:
-  metadata.gz: e45d48b488c7cc0bdea10fbea03397539a5533b1d98647aaf9f942150caaa121dc958bc57cc183782ffc7b74c98305a90ef14c9d07dfcb00b562372bf0c7048b
-  data.tar.gz: 23932d46b74e4f994b4e8a09d6b81e933f0798fa45608968e63f37b39476de0428c6315ebb925e18b043dabc4b10d0d8336e72d65407f61ba5e678c591d9918a
+  metadata.gz: 447ed6984427a1649f76ed733c2eafcf6324a5591265cd3f3ec9d29ca8f09552c2a93134b3a481ab06fd1d64b1f78db405fd4ade10e1b372eb62b6e8cbdfe364
+  data.tar.gz: c4b1823ae678c44d3d0892ff2bb44dc6ec844339717e1a8e0ce62ff564aee690e3f5d19c9ab254c6538ce2e3e4466c1789bc5b50beba787e49a8af22d60f892f

data/README.md

@@ -124,6 +124,59 @@ class Jobs::NoTransaction < QueueClassicPlus::Base
 end
 ```
 
+
+#### Callbacks
+
+Jobs support the following callbacks:
+```
+before_enqueue
+after_enqueue
+before_perform
+after_perform
+```
+
+- `enqueue` callbacks are called when the job is initially enqueued. Caveats:
+  - not called on retries
+  - not called when scheduled in the future (i.e. `Job.enqueue_perform_in`)
+  - still called if enqueuing fails because a job with `lock!` is already enqueued. 
+- `perform` callbacks are called any time the job is picked up and run by a worker, including retries and jobs scheduled in the future.
+
+Callbacks can either be implemented by providing a method to be called:
+
+```ruby
+class Jobs::WithCallbackMethods < QueueClassicPlus::Base
+  before_enqueue :before_enqueue_method
+
+  @queue = :low
+
+  def self.before_enqueue_method(*args)
+    # ...
+  end
+
+  def self.perform(user_id)
+    # ...
+  end
+end
+```
+
+or by providing a block:
+
+```ruby
+class Jobs::WithCallbackBlocks < QueueClassicPlus::Base
+  before_enqueue do |*args|
+    # ...
+  end
+
+  @queue = :low
+
+  def self.perform(user_id)
+    # ...
+  end
+end
+```
+
+The `args` passed to the callback method/block will be the same as the arguments passed to `QueueClassicPlus::Base.do` and `QueueClassicPlus::Base.perform`. 
+
 ## Advanced configuration
 
 If you want to log exceptions in your favorite exception tracker. You can configured it like so:

data/lib/queue_classic_plus/base.rb

@@ -52,8 +52,32 @@ module QueueClassicPlus
       QueueClassicPlus.logger
     end
 
+    def self.can_enqueue?(method, *args)
+      if locked?
+        max_lock_time = ENV.fetch("QUEUE_CLASSIC_MAX_LOCK_TIME", 10 * 60).to_i
+
+        q = "SELECT COUNT(1) AS count
+             FROM
+               (
+                 SELECT 1
+                 FROM queue_classic_jobs
+                 WHERE q_name = $1 AND method = $2 AND args::text = $3::text
+                   AND (locked_at IS NULL OR locked_at > current_timestamp - interval '#{max_lock_time} seconds')
+                 LIMIT 1
+               )
+             AS x"
+
+        result = QC.default_conn_adapter.execute(q, @queue, method, JSON.dump(serialized(args)))
+        result['count'].to_i == 0
+      else
+        true
+      end
+    end
+
     def self.enqueue(method, *args)
-      queue.enqueue(method, *serialized(args), lock: locked?)
+      if can_enqueue?(method, *args)
+        queue.enqueue(method, *serialized(args))
+      end
     end
 
     def self.enqueue_perform(*args)
@@ -112,6 +136,22 @@ module QueueClassicPlus
       end
     end
 
+    def self.before_enqueue(callback_method = nil, &block)
+      define_callback(:enqueue, :before, callback_method, &block)
+    end
+
+    def self.after_enqueue(callback_method = nil, &block)
+      define_callback(:enqueue, :after, callback_method, &block)
+    end
+
+    def self.before_perform(callback_method = nil, &block)
+      define_callback(:perform, :before, callback_method, &block)
+    end
+
+    def self.after_perform(callback_method = nil, &block)
+      define_callback(:perform, :after, callback_method, &block)
+    end
+
     # Debugging
     def self.list
       q = "SELECT * FROM queue_classic_jobs WHERE q_name = '#{@queue}'"
@@ -141,5 +181,26 @@ module QueueClassicPlus
     def self.execute(sql, *args)
       QC.default_conn_adapter.execute(sql, *args)
     end
+
+    def self.define_callback(name, type, callback_method, &_block)
+      singleton_class.prepend(
+        Module.new do |callback_module|
+          callback_module.define_method(name) do |*args|
+            callback_args = args
+              if name == :enqueue
+                callback_args = callback_args.drop(1) # Class/method is the first argument. Callbacks don't need that.
+              end
+
+              if type == :before
+                block_given? ? yield(*callback_args) : send(callback_method, *callback_args)
+              end
+              super(*args)
+              if type == :after
+                block_given? ? yield(*callback_args) : send(callback_method, *callback_args)
+              end
+          end
+        end
+      )
+    end
   end
 end

data/lib/queue_classic_plus/queue_classic/queue.rb

@@ -50,25 +50,5 @@ module QC
       end
     end
 
-    def enqueue(method, *args, lock: false)
-      QC.log_yield(:measure => 'queue.enqueue') do
-        insert_sql = <<-EOF
-          INSERT INTO #{QC.table_name} (q_name, method, args, lock)
-          VALUES ($1, $2, $3, $4)
-          ON CONFLICT (q_name, method, args) WHERE lock IS TRUE DO NOTHING
-          RETURNING id
-        EOF
-        begin
-          retries ||= 0
-          conn_adapter.execute(insert_sql, name, method, JSON.dump(args), lock)
-        rescue PG::Error => error
-          if (retries += 1) < 2
-            retry
-          else
-            raise
-          end
-        end
-      end
-    end
   end
 end

data/lib/queue_classic_plus/version.rb

@@ -1,3 +1,3 @@
 module QueueClassicPlus
-  VERSION = '4.0.0.alpha17'.freeze
+  VERSION = '4.0.0.alpha19'.freeze
 end

data/lib/queue_classic_plus.rb

@@ -15,18 +15,14 @@ module QueueClassicPlus
 
   def self.migrate(c = QC::default_conn_adapter.connection)
     conn = QC::ConnAdapter.new(connection: c)
-    conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN IF NOT EXISTS last_error TEXT")
-    conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN IF NOT EXISTS remaining_retries INTEGER")
-    conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN IF NOT EXISTS lock BOOLEAN NOT NULL DEFAULT FALSE")
-    conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS index_queue_classic_jobs_enqueue_lock on queue_classic_jobs(q_name, method, args) WHERE lock IS TRUE")
+    conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN last_error TEXT")
+    conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN remaining_retries INTEGER")
   end
 
   def self.demigrate(c = QC::default_conn_adapter.connection)
     conn = QC::ConnAdapter.new(connection: c)
-    conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN IF EXISTS last_error")
-    conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN IF EXISTS remaining_retries")
-    conn.execute("DROP INDEX IF EXISTS index_queue_classic_jobs_enqueue_lock")
-    conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN IF EXISTS lock")
+    conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN last_error")
+    conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN remaining_retries")
   end
 
   def self.exception_handler

data/spec/base_spec.rb

@@ -3,24 +3,6 @@ require 'active_record'
 
 describe QueueClassicPlus::Base do
   context "A child of QueueClassicPlus::Base" do
-    subject do
-      Class.new(QueueClassicPlus::Base) do
-        @queue = :test
-      end
-    end
-
-    it "allows multiple enqueues" do
-      threads = []
-      10.times do
-        threads << Thread.new do
-          subject.do
-        end
-      end
-      threads.each(&:join)
-
-      expect(subject).to have_queue_size_of(10)
-    end
-
     context "that is locked" do
       subject do
         Class.new(QueueClassicPlus::Base) do
@@ -30,28 +12,9 @@ describe QueueClassicPlus::Base do
       end
 
       it "does not allow multiple enqueues" do
-        threads = []
-        10.times do
-          threads << Thread.new do
-            subject.do
-            expect(subject).to have_queue_size_of(1)
-          end
-        end
-        threads.each(&:join)
-      end
-
-      it "allows enqueueing same job with different arguments" do
-        threads = []
-        (1..3).each do |arg|
-          10.times do
-            threads << Thread.new do
-              subject.do(arg)
-            end
-          end
-        end
-        threads.each(&:join)
-
-        expect(subject).to have_queue_size_of(3)
+        subject.do
+        subject.do
+        expect(subject).to have_queue_size_of(1)
       end
 
       it "checks for an existing job using the same serializing as job enqueuing" do
@@ -65,22 +28,13 @@ describe QueueClassicPlus::Base do
         subject.do(date)
         expect(subject).to have_queue_size_of(1)
       end
-    end
 
-    context "when in a transaction" do
-      subject do
-        Class.new(QueueClassicPlus::Base) do
-          @queue = :test
-          lock!
-        end
-      end
-
-      it "does not create another transaction when enqueueing" do
-        conn = QC.default_conn_adapter.connection
-        expect(conn).to receive(:transaction).exactly(1).times.and_call_original
-        conn.transaction do
-          subject.do
-        end
+      it "does allow multiple enqueues if something got locked for too long" do
+        subject.do
+        one_day_ago = Time.now - 60*60*24
+        execute "UPDATE queue_classic_jobs SET locked_at = '#{one_day_ago}' WHERE q_name = 'test'"
+        subject.do
+        expect(subject).to have_queue_size_of(2)
       end
     end
 
@@ -230,6 +184,172 @@ describe QueueClassicPlus::Base do
         subject._perform(42, true)
       end
     end
+
+    context "with callbacks defined" do
+      subject do
+        Class.new(QueueClassicPlus::Base) do
+          @queue = :test
+
+          before_enqueue :before_enqueue_method
+          after_enqueue :after_enqueue_method
+
+          before_perform :before_perform_method
+          after_perform :after_perform_method
+
+          def self.before_enqueue_method(*_args); end;
+          def self.after_enqueue_method(*_args); end;
+          def self.before_perform_method(*_args); end;
+          def self.after_perform_method(*_args); end;
+
+          def self.perform(*_args); end;
+        end
+      end
+
+      it "passes enqueue arguments to callbacks" do
+        expect(subject).to receive(:before_enqueue_method).with("enqueue_argument").once
+        expect(subject).to receive(:after_enqueue_method).with("enqueue_argument").once
+
+        subject.do("enqueue_argument")
+      end
+
+      it "passes perform arguments to callbacks" do
+        expect(subject).to receive(:before_perform_method).with("perform_argument").once
+        expect(subject).to receive(:after_perform_method).with("perform_argument").once
+
+        subject.perform("perform_argument")
+      end
+
+      context "when enqueued" do
+        it "calls the enqueue callback methods" do
+          expect(subject).to receive(:before_enqueue_method).once
+          expect(subject).to receive(:after_enqueue_method).once
+
+          subject.do
+        end
+
+        it "does not call the perform callbacks" do
+          expect(subject).to_not receive(:before_perform_method)
+          expect(subject).to_not receive(:after_perform_method)
+
+          subject.do
+        end
+      end
+
+      context "when perform" do
+        it "calls the perform callback methods" do
+          expect(subject).to receive(:before_perform_method).once
+          expect(subject).to receive(:after_perform_method).once
+
+          subject.perform
+        end
+
+        it "does not call the enqueue callbacks" do
+          expect(subject).to_not receive(:before_enqueue_method)
+          expect(subject).to_not receive(:after_enqueue_method)
+
+          subject.perform
+        end
+
+        context "when perform fails" do
+          subject do
+            Class.new(QueueClassicPlus::Base) do
+              @queue = :test
+
+              before_perform :before_perform_method
+              after_perform :after_perform_method
+
+              def self.before_perform_method(*_args); end;
+              def self.after_perform_method(*_args); end;
+
+              def self.perform(*_args)
+                raise StandardError
+              end
+            end
+          end
+
+          it "does not call after callbacks" do
+            expect(subject).to receive(:before_perform_method).once
+            expect(subject).to_not receive(:after_perform_method)
+
+            begin
+              subject.perform
+            rescue StandardError
+            end
+          end
+        end
+      end
+
+      context "when callback defined multiple times" do
+        subject do
+          Class.new(QueueClassicPlus::Base) do
+            @queue = :test
+
+            before_enqueue :before_enqueue_method_1
+            before_enqueue :before_enqueue_method_2
+            before_enqueue :before_enqueue_method_3
+
+            def self.before_enqueue_method_1(*_args); end;
+            def self.before_enqueue_method_2(*_args); end;
+            def self.before_enqueue_method_3(*_args); end;
+
+            def self.perform(*_args); end;
+          end
+        end
+
+        it "calls each callback" do
+          expect(subject).to receive(:before_enqueue_method_1).once
+          expect(subject).to receive(:before_enqueue_method_2).once
+          expect(subject).to receive(:before_enqueue_method_3).once
+
+          subject.do
+        end
+      end
+    end
+
+    context "with callback blocks defined" do
+      subject do
+        Class.new(QueueClassicPlus::Base) do
+          @queue = :test
+          class_variable_set(:@@block_result, [])
+
+          before_enqueue do |*_args|
+            class_variable_get(:@@block_result).append("before_enqueue_block")
+          end
+          after_enqueue do |*_args|
+            class_variable_get(:@@block_result).append("after_enqueue_block")
+          end
+
+          before_perform do |*_args|
+            class_variable_get(:@@block_result).append("before_perform_block")
+          end
+          after_perform do |*_args|
+            class_variable_get(:@@block_result).append("after_perform_block")
+          end
+
+          def self.perform; end;
+        end
+      end
+
+      context "when enqueued" do
+        it "calls the enqueue callback blocks" do
+          subject.do
+
+          expect(subject.class_variable_get(:@@block_result)).to eq(
+            %w(before_enqueue_block after_enqueue_block)
+          )
+        end
+      end
+
+      context "when perform" do
+        it "calls the perform callback blocks" do
+          subject.perform
+
+          expect(subject.class_variable_get(:@@block_result)).to eq(
+            %w(before_perform_block after_perform_block)
+          )
+        end
+      end
+    end
   end
 
   describe ".librato_key" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic_plus
 version: !ruby/object:Gem::Version
-  version: 4.0.0.alpha17
+  version: 4.0.0.alpha19
 platform: ruby
 authors:
 - Simon Mathieu
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-22 00:00:00.000000000 Z
+date: 2023-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: queue_classic