coil 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f7049380265432b66236482dba5ed20a432673b604021ad2ab0e1859f939bbf
-  data.tar.gz: 7160db6c58dff54ad63fd03898e7262b3ab6c8811d93fadec005c5dcfdfdbebf
+  metadata.gz: fb9891f8607d9006f5b72a07e88725a18e57dd6ea8a22c7c610eb9e490557983
+  data.tar.gz: eac76e2c7a1a1c3f3697613a637fa939b11ecb588e65dd828cb4ddd2f7b1137f
 SHA512:
-  metadata.gz: 4bc6ed0c1ab07e8afd6e9ae51baf9f61f162806b4f59dc727c155fd79082607e9181158096085db6ed8fdebb9adbeaf204ba92c8ddd9b6a245b4c0d4e85c9613
-  data.tar.gz: 880a4cf79ecf5b761ee4dac86c984e36387d15479c5e0d51f58c370bd0301dd6f448aeeebfa6744d42ed3faecd67ce8983d938988d4dd5be209d713925f91fa9
+  metadata.gz: 0c23e1a776d61ce557f1ace11aac2dc29fc71ee87a4dfd8037bb50266bdbb1254d87d5e1a3870f1622f9aa7837719e0608a89f0162da0f6c55e64a7adaa21341
+  data.tar.gz: 961ef82b7218976a207cf06af86ca2e7339fc31d255a6d1aaf8d80bf762436afe88e43af3dc0f27e69950fe0d5cb4abdfd5350bc17596aa05000d85b5878cc83

data/README.md

@@ -36,10 +36,11 @@ gem "schema_version_cache"
 
 Install engine and migrations:
 ```console
-$ bundle
-$ bundle exec rails coil:install:migrations
-$ bundle exec rails db:migrate
+$ bundle install
+$ bundle exec rails coil:install:migrations db:migrate
 ```
+(_NOTE_: Also run the above commands when upgrading, as newer versions may
+introduce additional migrations.)
 
 Register periodic jobs:
 ```ruby
@@ -49,9 +50,15 @@ Sidekiq.configure_server do |config|
   config.periodic do |mgr|
     mgr.register("*/10 * * * *", "Coil::Inbox::MessagesPeriodicJob")
     mgr.register("5-59/10 * * * *", "Coil::Outbox::MessagesPeriodicJob")
+
+    mgr.register("7-59/20 * * * *", "Coil::Inbox::MessagesCleanupJob")
+    mgr.register("12-59/20 * * * *", "Coil::Outbox::MessagesCleanupJob")
   end
 end
 ```
+(_NOTE_: The cleanup jobs delete already-processed messages once their retention
+period has passed. Retention periods can be [configured](#configuration) using
+`Coil.inbox_retention_period` and `Coil.outbox_retention_period`.)
 
 Filter retryable errors out of alerting, e.g. airbrake:
 ```ruby
@@ -242,6 +249,8 @@ initializer at `config/initializers/coil.rb` with the following content, then
 uncomment and adjust the settings you wish to change:
 ```ruby
 # Coil.sidekiq_queue = "default"
+# Coil.inbox_retention_period = 12.weeks
+# Coil.outbox_retention_period = 12.weeks
 ```
 
 ## Development

data/app/jobs/coil/inbox/messages_cleanup_job.rb

@@ -0,0 +1,18 @@
+# typed: strict
+
+# This job deletes processed inbox messages whose retention period has passed.
+module Coil
+  module Inbox
+    class MessagesCleanupJob < TransactionalMessagesCleanupJob
+      private
+
+      def message_parent_class
+        ::Coil::Inbox::Message
+      end
+
+      def retention_period
+        ::Coil.inbox_retention_period
+      end
+    end
+  end
+end

data/app/jobs/coil/outbox/messages_cleanup_job.rb

@@ -0,0 +1,18 @@
+# typed: strict
+
+# This job deletes processed outbox messages whose retention period has passed.
+module Coil
+  module Outbox
+    class MessagesCleanupJob < TransactionalMessagesCleanupJob
+      private
+
+      def message_parent_class
+        ::Coil::Outbox::Message
+      end
+
+      def retention_period
+        ::Coil.outbox_retention_period
+      end
+    end
+  end
+end

data/app/jobs/coil/transactional_messages_cleanup_job.rb

@@ -0,0 +1,106 @@
+# typed: strict
+# frozen_string_literal: true
+
+# A cleanup job deletes processed messages whose retention period has passed.
+module Coil
+  class TransactionalMessagesCleanupJob < ApplicationJob
+    DuplicateJobError = Class.new(StandardError)
+
+    # Sidekiq is not designed for long-running jobs, so we place an upper bound
+    # on job duration. When a job exceeds this bound, we'll enqueue a subsequent
+    # job to pick up where we left off.
+    MAX_DURATION = 5.minutes
+
+    sidekiq_options retry: 4, dead: false
+
+    def perform(batch_size = 1000)
+      result = delete_messages(batch_size)
+      total_deletions = result.deletions.values.sum
+      deletions_json = result.deletions.to_json
+
+      case result
+      when Finished
+        Rails.logger.info(<<~INFO.squish)
+          #{self.class} finished after deleting #{total_deletions} messages
+          (#{deletions_json}).
+        INFO
+      when ExceededDeadline
+        Rails.logger.info(<<~INFO.squish)
+          #{self.class} exceeded deadline after deleting #{total_deletions}
+          messages (#{deletions_json}). Enqueuing subsequent job.
+        INFO
+        self.class.perform_async(batch_size)
+      end
+    rescue DuplicateJobError
+      # A duplicate job is in the midst of its message-deletion loop. We'll call
+      # this job done and allow the other one to continue.
+    end
+
+    private
+
+    def delete_messages(batch_size)
+      ApplicationRecord.uncached do
+        locking do
+          _delete_messages(batch_size)
+        end
+      end
+    end
+
+    Finished = Struct.new(:deletions)
+    ExceededDeadline = Struct.new(:deletions)
+
+    def _delete_messages(batch_size)
+      now = Time.current
+      created_before = now - retention_period
+      deadline = now + MAX_DURATION
+      deletions = Hash.new(0)
+
+      # Identify distinct message types, their associated job types, and the
+      # messages that can safely be deleted. Delete in batches until finished
+      # or the deadline is exceeded.
+      message_parent_class.select(:type).distinct.pluck(:type).each do |type|
+        message_class = message_class_for(type)
+        messages =
+          if message_class.present?
+            message_class.processed(processor_name: message_class.new.job_class.name)
+          else
+            message_parent_class.where(type:)
+          end
+
+        messages
+          .where(created_at: nil...created_before)
+          .in_batches(of: batch_size) do |batch|
+            return ExceededDeadline.new(deletions) if Time.current > deadline
+            deletions[type] += batch.distinct(false).delete_all
+          end
+      end
+
+      Finished.new(deletions)
+    end
+
+    QUEUE_TYPE = "CLEANUP_QUEUE"
+
+    def locking(&blk)
+      QueueLocking.locking(
+        queue_type: QUEUE_TYPE,
+        message_type: message_parent_class.to_s,
+        message_keys: [self.class.to_s],
+        wait: false,
+        &blk
+      )
+    rescue QueueLocking::LockWaitTimeout
+      raise DuplicateJobError
+    end
+
+    def message_class_for(type)
+      message_parent_class.sti_class_for(type)
+    rescue ActiveRecord::SubclassNotFound
+    end
+
+    def message_parent_class
+    end
+
+    def retention_period
+    end
+  end
+end

data/db/migrate/20250101182458_add_foreign_key_to_inbox_completions_on_delete_cascade.rb

@@ -0,0 +1,32 @@
+# typed: false
+
+class AddForeignKeyToInboxCompletionsOnDeleteCascade < ActiveRecord::Migration[6.0]
+  def change
+    from_table = :coil_inbox_completions
+    to_table = :coil_inbox_messages
+    column = :last_completed_message_id
+    old_key = find_foreign_key(from_table, to_table:, column:, on_delete: nil)
+
+    # NOTE: To minimize the impact on read/write availability while adding this
+    # foreign key, we specify `validate: false`, skipping the table scan that
+    # would normally be performed to validate that all existing rows satisfy the
+    # new constraint. We can then validate it in a separate transaction (see the
+    # next migration) without blocking reads/writes to the tables involved.
+    # https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DESC-ADD-TABLE-CONSTRAINT
+    add_foreign_key(
+      from_table,
+      to_table,
+      column:,
+      name: "#{old_key.name}_on_delete_cascade",
+      on_delete: :cascade,
+      validate: false
+    )
+  end
+
+  private
+
+  def find_foreign_key(from_table, **options)
+    foreign_keys(from_table).detect { |fk| fk.defined_for?(**options) } ||
+      raise("No foreign key found from table '#{from_table}' for #{options}")
+  end
+end

data/db/migrate/20250102031753_validate_foreign_key_inbox_completions.rb

@@ -0,0 +1,19 @@
+# typed: false
+
+class ValidateForeignKeyInboxCompletions < ActiveRecord::Migration[6.0]
+  def change
+    from_table = :coil_inbox_completions
+    to_table = :coil_inbox_messages
+    column = :last_completed_message_id
+    key = find_foreign_key(from_table, to_table:, column:, on_delete: :cascade)
+
+    validate_foreign_key from_table, to_table, name: key.name
+  end
+
+  private
+
+  def find_foreign_key(from_table, **options)
+    foreign_keys(from_table).detect { |fk| fk.defined_for?(**options) } ||
+      raise("No foreign key found from table '#{from_table}' for #{options}")
+  end
+end

data/db/migrate/20250102040148_remove_old_foreign_key_from_inbox_completions.rb

@@ -0,0 +1,24 @@
+# typed: false
+
+class RemoveOldForeignKeyFromInboxCompletions < ActiveRecord::Migration[6.0]
+  def change
+    from_table = :coil_inbox_completions
+    to_table = :coil_inbox_messages
+    column = :last_completed_message_id
+    new_key = find_foreign_key(from_table, to_table:, column:, on_delete: :cascade)
+
+    remove_foreign_key(
+      from_table,
+      to_table,
+      column:,
+      name: new_key.name.delete_suffix("_on_delete_cascade")
+    )
+  end
+
+  private
+
+  def find_foreign_key(from_table, **options)
+    foreign_keys(from_table).detect { |fk| fk.defined_for?(**options) } ||
+      raise("No foreign key found from table '#{from_table}' for #{options}")
+  end
+end

data/db/migrate/20250102040649_add_foreign_key_to_outbox_completions_on_delete_cascade.rb

@@ -0,0 +1,32 @@
+# typed: false
+
+class AddForeignKeyToOutboxCompletionsOnDeleteCascade < ActiveRecord::Migration[6.0]
+  def change
+    from_table = :coil_outbox_completions
+    to_table = :coil_outbox_messages
+    column = :last_completed_message_id
+    old_key = find_foreign_key(from_table, to_table:, column:, on_delete: nil)
+
+    # NOTE: To minimize the impact on read/write availability while adding this
+    # foreign key, we specify `validate: false`, skipping the table scan that
+    # would normally be performed to validate that all existing rows satisfy the
+    # new constraint. We can then validate it in a separate transaction (see the
+    # next migration) without blocking reads/writes to the tables involved.
+    # https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DESC-ADD-TABLE-CONSTRAINT
+    add_foreign_key(
+      from_table,
+      to_table,
+      column:,
+      name: "#{old_key.name}_on_delete_cascade",
+      on_delete: :cascade,
+      validate: false
+    )
+  end
+
+  private
+
+  def find_foreign_key(from_table, **options)
+    foreign_keys(from_table).detect { |fk| fk.defined_for?(**options) } ||
+      raise("No foreign key found from table '#{from_table}' for #{options}")
+  end
+end

data/db/migrate/20250102040950_validate_foreign_key_outbox_completions.rb

@@ -0,0 +1,19 @@
+# typed: false
+
+class ValidateForeignKeyOutboxCompletions < ActiveRecord::Migration[6.0]
+  def change
+    from_table = :coil_outbox_completions
+    to_table = :coil_outbox_messages
+    column = :last_completed_message_id
+    key = find_foreign_key(from_table, to_table:, column:, on_delete: :cascade)
+
+    validate_foreign_key from_table, to_table, name: key.name
+  end
+
+  private
+
+  def find_foreign_key(from_table, **options)
+    foreign_keys(from_table).detect { |fk| fk.defined_for?(**options) } ||
+      raise("No foreign key found from table '#{from_table}' for #{options}")
+  end
+end

data/db/migrate/20250102041225_remove_old_foreign_key_from_outbox_completions.rb

@@ -0,0 +1,24 @@
+# typed: false
+
+class RemoveOldForeignKeyFromOutboxCompletions < ActiveRecord::Migration[6.0]
+  def change
+    from_table = :coil_outbox_completions
+    to_table = :coil_outbox_messages
+    column = :last_completed_message_id
+    new_key = find_foreign_key(from_table, to_table:, column:, on_delete: :cascade)
+
+    remove_foreign_key(
+      from_table,
+      to_table,
+      column:,
+      name: new_key.name.delete_suffix("_on_delete_cascade")
+    )
+  end
+
+  private
+
+  def find_foreign_key(from_table, **options)
+    foreign_keys(from_table).detect { |fk| fk.defined_for?(**options) } ||
+      raise("No foreign key found from table '#{from_table}' for #{options}")
+  end
+end

data/lib/coil/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Coil
-  VERSION = "1.4.0"
+  VERSION = "1.5.0"
 end

data/lib/coil.rb

@@ -7,4 +7,6 @@ require "coil/queue_locking"
 
 module Coil
   mattr_accessor :sidekiq_queue, default: "default"
+  mattr_accessor :inbox_retention_period, default: 12.weeks
+  mattr_accessor :outbox_retention_period, default: 12.weeks
 end

data/rbi/coil.rbi

@@ -79,6 +79,16 @@ Coil::Inbox::Message::COMPLETION = Coil::Inbox::Completion
 Coil::Inbox::Message::PERSISTENCE_QUEUE = T.let(T.unsafe(nil), String)
 Coil::Inbox::Message::PROCESS_QUEUE = T.let(T.unsafe(nil), String)
 
+class Coil::Inbox::MessagesCleanupJob < ::Coil::TransactionalMessagesCleanupJob
+  private
+
+  sig { override.returns(T.class_of(::Coil::Inbox::Message)) }
+  def message_parent_class; end
+
+  sig { override.returns(ActiveSupport::Duration) }
+  def retention_period; end
+end
+
 class Coil::Inbox::MessagesPeriodicJob < ::Coil::TransactionalMessagesPeriodicJob
   private
 
@@ -140,6 +150,16 @@ Coil::Outbox::Message::COMPLETION = Coil::Outbox::Completion
 Coil::Outbox::Message::PERSISTENCE_QUEUE = T.let(T.unsafe(nil), String)
 Coil::Outbox::Message::PROCESS_QUEUE = T.let(T.unsafe(nil), String)
 
+class Coil::Outbox::MessagesCleanupJob < ::Coil::TransactionalMessagesCleanupJob
+  private
+
+  sig { override.returns(T.class_of(::Coil::Outbox::Message)) }
+  def message_parent_class; end
+
+  sig { override.returns(ActiveSupport::Duration) }
+  def retention_period; end
+end
+
 class Coil::Outbox::MessagesPeriodicJob < ::Coil::TransactionalMessagesPeriodicJob
   private
 
@@ -311,6 +331,45 @@ class Coil::TransactionalMessagesJob::DuplicateJobError < ::StandardError; end
 Coil::TransactionalMessagesJob::MAX_DURATION = T.let(T.unsafe(nil), ActiveSupport::Duration)
 class Coil::TransactionalMessagesJob::RetryableError < ::StandardError; end
 
+class Coil::TransactionalMessagesCleanupJob < ::Coil::ApplicationJob
+  abstract!
+
+  class DuplicateJobError < ::StandardError; end
+
+  MAX_DURATION = T.let(T.unsafe(nil), ActiveSupport::Duration)
+
+  sig { params(batch_size: Integer).void }
+  def perform(batch_size = 1000); end
+
+  private
+
+  Result = T.type_alias { T.any(Finished, ExceededDeadline) }
+
+  sig { params(batch_size: Integer).returns(Result) }
+  def delete_messages(batch_size); end
+
+  sig { params(batch_size: Integer).returns(Result) }
+  def _delete_messages(batch_size); end
+
+  QUEUE_TYPE = T.let(T.unsafe(nil), String)
+
+  sig {
+    type_parameters(:P)
+      .params(blk: T.proc.returns(T.type_parameter(:P)))
+      .returns(T.type_parameter(:P))
+  }
+  def locking(&blk); end
+
+  sig { params(type: String).returns(T.nilable(::Coil::AnyMessageClass)) }
+  def message_class_for(type); end
+
+  sig { abstract.returns(::Coil::AnyMessageClass) }
+  def message_parent_class; end
+
+  sig { abstract.returns(ActiveSupport::Duration) }
+  def retention_period; end
+end
+
 class Coil::TransactionalMessagesPeriodicJob < ::Coil::ApplicationJob
   abstract!
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coil
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Ivan Brennan
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-11 00:00:00.000000000 Z
+date: 2025-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -253,8 +253,11 @@ files:
 - README.md
 - Rakefile
 - app/jobs/coil/application_job.rb
+- app/jobs/coil/inbox/messages_cleanup_job.rb
 - app/jobs/coil/inbox/messages_periodic_job.rb
+- app/jobs/coil/outbox/messages_cleanup_job.rb
 - app/jobs/coil/outbox/messages_periodic_job.rb
+- app/jobs/coil/transactional_messages_cleanup_job.rb
 - app/jobs/coil/transactional_messages_job.rb
 - app/jobs/coil/transactional_messages_periodic_job.rb
 - app/models/coil/application_record.rb
@@ -268,6 +271,12 @@ files:
 - app/models/concerns/coil/transactional_message.rb
 - config/routes.rb
 - db/migrate/20240604163650_create_coil_tables.rb
+- db/migrate/20250101182458_add_foreign_key_to_inbox_completions_on_delete_cascade.rb
+- db/migrate/20250102031753_validate_foreign_key_inbox_completions.rb
+- db/migrate/20250102040148_remove_old_foreign_key_from_inbox_completions.rb
+- db/migrate/20250102040649_add_foreign_key_to_outbox_completions_on_delete_cascade.rb
+- db/migrate/20250102040950_validate_foreign_key_outbox_completions.rb
+- db/migrate/20250102041225_remove_old_foreign_key_from_outbox_completions.rb
 - lib/coil.rb
 - lib/coil/engine.rb
 - lib/coil/queue_locking.rb