coil 1.6.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6585ea0b663bc817f16b5a10f4b63dcbc7978b7a4f869468c33706672cb1204d
-  data.tar.gz: d0cb7da9194256b429eceaae88f1a0860244da40c980470caf5caf9dfa0cf022
+  metadata.gz: 4d8cfb14efec96e90e76ae65f4dc955cbfc6096dd2769bffd96cec917170cc7f
+  data.tar.gz: 412d34f5d3692749ac166b81a3560a2c2c95b626b4aee8aa8e65cc44c264b4ba
 SHA512:
-  metadata.gz: 7e7c8a7b98ed57bb9d25bca7fbdbfeb88ae14bc331d24494eb1964cd9eab723e6de59093bc0c6c02727b452b4d0b3753963600317b625d7ef4971dd907d9a0f5
-  data.tar.gz: 958f4e1ad44cb749365d581cb92bf2c2864363aae0bf0b9e17254a660c1f176077d67a5a65c3f87f67d1203c1cf0ce1a9d2c085cc74ba31641517109ca580f1d
+  metadata.gz: 30330b66219bccd62fbceef0716efc568540ca432464c94b81b3b09e0bc4d216e41a9e8e6c4e3e9bc46ad0de6af337e382f57cd34872990895d18d14ecbc7424
+  data.tar.gz: a89bf2e70d720a95d47e6157567e469374d53be4e580603aed02bae0ed7f70a28bae5b3270d7c1628b253eff1c44e4a02dc88d256cf97a8b7acab10972376d58

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## main
 
+## [1.7.0](https://github.com/OdekoTeam/coil/compare/1.6.1...1.7.0) (2026-03-16)
+#### Changed
+- Improve retry policy ([d51ea55](https://github.com/OdekoTeam/coil/commit/d51ea553eaa41cb99a4da70a0316707780f1a79d))
+  - **NOTE**: this release includes additional migrations, so **be sure to run** `bundle exec rails coil:install:migrations db:migrate`
+
 ## [1.6.1](https://github.com/OdekoTeam/coil/compare/1.6.0...1.6.1) (2026-03-16)
 #### Fix
 - Fix migration version ([63866a2](https://github.com/OdekoTeam/coil/commit/63866a21d0d54080b2ef8a37046237524b281089))

data/app/jobs/coil/transactional_messages_job.rb

@@ -10,8 +10,6 @@ module Coil
     # job to pick up where we left off.
     MAX_DURATION = 5.minutes
 
-    sidekiq_options retry: 4, dead: false
-
     def perform(key, processor_name = self.class.to_s)
       deadline = Time.current + MAX_DURATION
       next_in_line = process_messages(key:, processor_name:, deadline:)
@@ -38,18 +36,40 @@ module Coil
         next_in_line = next_message(key:, processor_name:)
         return next_in_line if next_in_line.nil? || Time.current > deadline
 
-        locking(key) do
-          message = next_message(key:, processor_name:)
+        outcome = process_next(key:, processor_name:)
+        raise outcome.error if outcome.error
+      end
+    end
+
+    class Outcome
+      attr_accessor :error
+    end
+
+    def process_next(key:, processor_name:)
+      outcome = Outcome.new
 
-          if message.present?
-            around_process(message, processor_name:) do
-              pre_process(message)
-              message.processed(processor_name:)
-              process(message)
+      locking(key) do
+        message = next_message(key:, processor_name:)
+
+        if message.present?
+          message.increment!(:processor_attempts)
+          begin
+            # Use a nested transaction and deferred error-handling to ensure the
+            # processor_attempts increment persists, even if the attempt fails.
+            ApplicationRecord.transaction(requires_new: true) do
+              around_process(message, processor_name:) do
+                pre_process(message)
+                message.processed(processor_name:)
+                process(message)
+              end
             end
+          rescue => e
+            outcome.error = e
           end
         end
       end
+
+      outcome
     end
 
     def around_process(message, processor_name:, &blk)

data/app/jobs/coil/transactional_messages_periodic_job.rb

@@ -1,17 +1,28 @@
 # typed: strict
 
+require "sidekiq/api"
+
 # The periodic job acts as a fallback mechanism, polling for messages that were
 # not enqueued and processed automatically upon create, e.g. due to a failure to
 # push a job onto the Redis queue.
 module Coil
   class TransactionalMessagesPeriodicJob < ApplicationJob
+    ATTEMPTS_THRESHOLD = 3
+
     def perform
-      t = Time.current - TransactionalMessagesJob::MAX_DURATION
+      q = Sidekiq::Queue.new(Coil.sidekiq_queue)
+      t = Time.current - q.latency - TransactionalMessagesJob::MAX_DURATION
 
       # Identify distinct message types, their associated job types, and
-      # the distinct keys for which we have unprocessed messages (excluding
-      # very recent messages, since a TransactionalMessagesJob may still be
-      # processing those). Then, enqueue the necessary jobs for those keys.
+      # the distinct keys for which we have unprocessed messages.
+      #
+      # Exclude very recent messages, since a TransactionalMessagesJob could
+      # still be processing those.
+      #
+      # Exclude keys where a processor has already initiated several attempts,
+      # since that's a strong indicator that automatic retries are in play.
+      #
+      # Then, enqueue the appropriate jobs.
       message_parent_class.select(:type).distinct.pluck(:type).each do |type|
         message_class = message_class_for(type)
         next unless message_class.present?
@@ -20,7 +31,8 @@ module Coil
         message_class
           .unprocessed(processor_name: job_class.name)
           .where(created_at: nil...t)
-          .distinct
+          .group(:key)
+          .having("MAX(processor_attempts) < ?", ATTEMPTS_THRESHOLD)
           .pluck(:key)
           .each { |k| job_class.perform_async(k) }
       end

data/db/migrate/20260314134950_add_processor_attempts_to_inbox_messages.rb

@@ -0,0 +1,18 @@
+# typed: false
+
+class AddProcessorAttemptsToInboxMessages < ActiveRecord::Migration[6.0]
+  def change
+    comment = <<~DOC.squish
+      Number of processor attempts that have been initiated on this message.
+    DOC
+
+    add_column(
+      :coil_inbox_messages,
+      :processor_attempts,
+      :integer,
+      null: false,
+      default: 0,
+      comment:
+    )
+  end
+end

data/db/migrate/20260314162532_add_processor_attempts_to_outbox_messages.rb

@@ -0,0 +1,18 @@
+# typed: false
+
+class AddProcessorAttemptsToOutboxMessages < ActiveRecord::Migration[6.0]
+  def change
+    comment = <<~DOC.squish
+      Number of processor attempts that have been initiated on this message.
+    DOC
+
+    add_column(
+      :coil_outbox_messages,
+      :processor_attempts,
+      :integer,
+      null: false,
+      default: 0,
+      comment:
+    )
+  end
+end

data/lib/coil/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Coil
-  VERSION = "1.6.1"
+  VERSION = "1.7.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coil
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Ivan Brennan
@@ -262,6 +262,8 @@ files:
 - db/migrate/20250102041225_remove_old_foreign_key_from_outbox_completions.rb
 - db/migrate/20260309171521_index_inbox_messages_on_type.rb
 - db/migrate/20260309173914_index_outbox_messages_on_type.rb
+- db/migrate/20260314134950_add_processor_attempts_to_inbox_messages.rb
+- db/migrate/20260314162532_add_processor_attempts_to_outbox_messages.rb
 - lib/coil.rb
 - lib/coil/engine.rb
 - lib/coil/queue_locking.rb