coil 1.3.0

data/app/models/coil/inbox/message.rb

@@ -0,0 +1,28 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Coil
+  module Inbox
+    class Message < ApplicationRecord
+      PERSISTENCE_QUEUE = "INBOX_PERSISTENCE_QUEUE"
+      PROCESS_QUEUE = "INBOX_PROCESS_QUEUE"
+
+      COMPLETION = ::Coil::Inbox::Completion
+
+      include TransactionalMessage
+
+      after_commit :enqueue_job, on: :create
+
+      def enqueue_job(processor_name = job_class.to_s)
+        perform_job_in(0.seconds, processor_name)
+      end
+
+      def perform_job_in(interval, processor_name = job_class.to_s)
+        job_class.perform_in(interval, key, processor_name)
+      end
+
+      def job_class
+      end
+    end
+  end
+end

data/app/models/coil/inbox.rb

@@ -0,0 +1,9 @@
+# typed: strict
+
+module Coil
+  module Inbox
+    def self.table_name_prefix
+      module_parent.table_name_prefix + "inbox_"
+    end
+  end
+end

data/app/models/coil/outbox/completion.rb

@@ -0,0 +1,11 @@
+# typed: strict
+
+module Coil
+  module Outbox
+    class Completion < ApplicationRecord
+      include PreventDestruction
+
+      attr_readonly :processor_name, :message_type, :message_key
+    end
+  end
+end

data/app/models/coil/outbox/message.rb

@@ -0,0 +1,28 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Coil
+  module Outbox
+    class Message < ApplicationRecord
+      PERSISTENCE_QUEUE = "OUTBOX_PERSISTENCE_QUEUE"
+      PROCESS_QUEUE = "OUTBOX_PROCESS_QUEUE"
+
+      COMPLETION = ::Coil::Outbox::Completion
+
+      include TransactionalMessage
+
+      after_commit :enqueue_job, on: :create
+
+      def enqueue_job(processor_name = job_class.to_s)
+        perform_job_in(0.seconds, processor_name)
+      end
+
+      def perform_job_in(interval, processor_name = job_class.to_s)
+        job_class.perform_in(interval, key, processor_name)
+      end
+
+      def job_class
+      end
+    end
+  end
+end

data/app/models/coil/outbox.rb

@@ -0,0 +1,9 @@
+# typed: strict
+
+module Coil
+  module Outbox
+    def self.table_name_prefix
+      module_parent.table_name_prefix + "outbox_"
+    end
+  end
+end

data/app/models/concerns/coil/prevent_destruction.rb

@@ -0,0 +1,24 @@
+# typed: false
+
+module Coil
+  module PreventDestruction
+    extend ActiveSupport::Concern
+
+    included do
+      unless self < ActiveRecord::Base
+        raise <<~ERR.squish
+          Cannot include PreventDestruction in #{self} because #{self} does not
+          inherit from ActiveRecord::Base.
+        ERR
+      end
+
+      before_destroy :prevent_destruction, prepend: true
+    end
+
+    private
+
+    def prevent_destruction
+      throw(:abort)
+    end
+  end
+end

data/app/models/concerns/coil/transactional_message.rb

@@ -0,0 +1,136 @@
+# typed: false
+
+module Coil
+  module TransactionalMessage
+    extend ActiveSupport::Concern
+
+    included do
+      unless self < ActiveRecord::Base
+        raise <<~ERR.squish
+          Cannot include TransactionalMessage in #{self} because #{self} does not
+          inherit from ActiveRecord::Base.
+        ERR
+      end
+
+      unless respond_to?(:sti_name)
+        raise <<~ERR.squish
+          Cannot include TransactionalMessage in #{self} because #{self} does not
+          use single table inheritance.
+        ERR
+      end
+
+      unless const_defined?(:COMPLETION)
+        raise <<~ERR.squish
+          Cannot include TransactionalMessage in #{self} because
+          #{self}::COMPLETION is not defined.
+        ERR
+      end
+
+      unless self::COMPLETION < ActiveRecord::Base
+        raise <<~ERR.squish
+          Cannot include TransactionalMessage in #{self} because
+          #{self}::COMPLETION does not inherit from ActiveRecord::Base.
+        ERR
+      end
+
+      unless const_defined?(:PERSISTENCE_QUEUE)
+        raise <<~ERR.squish
+          Cannot include TransactionalMessage in #{self} because
+          #{self}::PERSISTENCE_QUEUE is not defined.
+        ERR
+      end
+
+      unless const_defined?(:PROCESS_QUEUE)
+        raise <<~ERR.squish
+          Cannot include TransactionalMessage in #{self} because
+          #{self}::PROCESS_QUEUE is not defined.
+        ERR
+      end
+
+      validates :key, presence: true
+
+      attr_readonly :type, :key, :value
+
+      around_save :locking_persistence_queue, if: :new_record?
+
+      include PreventDestruction
+    end
+
+    def processed?(processor_name:)
+      persisted? &&
+        completions.exists?(processor_name:, last_completed_message_id: id...)
+    end
+
+    def processed(processor_name:)
+      c = completions.find_or_initialize_by(processor_name:)
+      max_id = [c.last_completed_message_id, id].compact.max
+      c.update!(last_completed_message_id: max_id)
+    end
+
+    def unprocessed_predecessors(processor_name:)
+      self.class.unprocessed(processor_name:).where(id: ...id, key:)
+    end
+
+    def locking_persistence_queue(wait: true, &blk)
+      self.class.locking_persistence_queue(keys: [key], wait:, &blk)
+    end
+
+    class_methods do
+      # Messages already processed by the given processor_name. If processor_name is
+      # an array, include messages already processed by any of the named processors.
+      def processed(processor_name:)
+        distinct.joins(
+          sanitize_sql([<<~SQL.squish, {processor_names: Array(processor_name)}])
+            INNER JOIN "#{self::COMPLETION.table_name}" "completions"
+              ON "completions"."processor_name" IN (:processor_names)
+             AND "completions"."message_type" = "#{table_name}"."type"
+             AND "completions"."message_key" = "#{table_name}"."key"
+             AND "completions"."last_completed_message_id" >= "#{table_name}"."id"
+          SQL
+        )
+      end
+
+      # Messages not yet processed by the given processor_name. If processor_name is
+      # an array, exclude messages already processed by any of the named processors.
+      def unprocessed(processor_name:)
+        joins(
+          sanitize_sql([<<~SQL.squish, {processor_names: Array(processor_name)}])
+            LEFT JOIN "#{self::COMPLETION.table_name}" "completions"
+              ON "completions"."processor_name" IN (:processor_names)
+             AND "completions"."message_type" = "#{table_name}"."type"
+             AND "completions"."message_key" = "#{table_name}"."key"
+             AND "completions"."last_completed_message_id" >= "#{table_name}"."id"
+          SQL
+        ).where(completions: {id: nil})
+      end
+
+      # Find the next message, with a given key, that hasn't already been processed
+      # by the given processor_name. If processor_name is an array, exclude messages
+      # already processed by any of the named processors.
+      def next_in_line(key:, processor_name:)
+        unprocessed(processor_name:).where(key:).order(id: :asc).first
+      end
+
+      def locking_persistence_queue(keys:, wait: true, &blk)
+        queue_type = self::PERSISTENCE_QUEUE
+        message_type = sti_name
+        QueueLocking.locking(queue_type:, message_type:, message_keys: keys, wait:, &blk)
+      end
+
+      def locking_process_queue(keys:, wait: true, &blk)
+        queue_type = self::PROCESS_QUEUE
+        message_type = sti_name
+        QueueLocking.locking(queue_type:, message_type:, message_keys: keys, wait:, &blk)
+      end
+    end
+
+    private
+
+    def completions
+      self.class::COMPLETION.where(
+        message_type: type,
+        message_key: key
+      )
+    end
+  end
+end

data/config/routes.rb

@@ -0,0 +1,4 @@
+# typed: false
+
+Coil::Engine.routes.draw do
+end

data/db/migrate/20240604163650_create_coil_tables.rb

@@ -0,0 +1,85 @@
+# typed: false
+
+class CreateCoilTables < ActiveRecord::Migration[6.0]
+  def change
+    create_table :coil_inbox_messages do |t|
+      t.string :type, null: false
+      t.jsonb :key, null: false
+      t.jsonb :value, null: false
+      t.jsonb :metadata, null: false, default: {}
+      t.timestamps
+
+      t.index [:type, :key]
+      t.index :created_at
+    end
+
+    create_table :coil_inbox_completions do |t|
+      t.string :processor_name, null: false
+      t.string :message_type, null: false
+      t.jsonb :message_key, null: false
+      t.references(
+        :last_completed_message,
+        null: false,
+        foreign_key: {to_table: :coil_inbox_messages},
+        index: true
+      )
+      t.timestamps
+
+      t.index(
+        [:processor_name, :message_type, :message_key],
+        name: "index_coil_inbox_completions_processor_message_type_key_uniq",
+        unique: true
+      )
+      # Although the above unique index means there can only be a single row for
+      # a given (processor_name, message_type, message_key) combo, providing an
+      # additional non-unique index that includes last_completed_message_id will
+      # allow some critical queries to leverage an index-only-scan (where the
+      # index itself provides all the required data, eliminating the need to
+      # also visit the table itself).
+      t.index(
+        [:processor_name, :message_type, :message_key, :last_completed_message_id],
+        name: "index_coil_inbox_completions_on_processor_message_completed"
+      )
+    end
+
+    create_table :coil_outbox_messages do |t|
+      t.string :type, null: false
+      t.jsonb :key, null: false
+      t.jsonb :value, null: false
+      t.jsonb :metadata, null: false, default: {}
+      t.timestamps
+
+      t.index [:type, :key]
+      t.index :created_at
+    end
+
+    create_table :coil_outbox_completions do |t|
+      t.string :processor_name, null: false
+      t.string :message_type, null: false
+      t.jsonb :message_key, null: false
+      t.references(
+        :last_completed_message,
+        null: false,
+        foreign_key: {to_table: :coil_outbox_messages},
+        index: true
+      )
+      t.timestamps
+
+      t.index(
+        [:processor_name, :message_type, :message_key],
+        name: "index_coil_outbox_completions_processor_message_type_key_uniq",
+        unique: true
+      )
+      # Although the above unique index means there can only be a single row for
+      # a given (processor_name, message_type, message_key) combo, providing an
+      # additional non-unique index that includes last_completed_message_id will
+      # allow some critical queries to leverage an index-only-scan (where the
+      # index itself provides all the required data, eliminating the need to
+      # also visit the table itself).
+      t.index(
+        [:processor_name, :message_type, :message_key, :last_completed_message_id],
+        name: "index_coil_outbox_completions_on_processor_message_completed"
+      )
+    end
+  end
+end

data/lib/coil/engine.rb

@@ -0,0 +1,9 @@
+# typed: strict
+
+module Coil
+  class Engine < ::Rails::Engine
+    isolate_namespace Coil
+    config.generators.api_only = true
+    config.generators.test_framework = :rspec
+  end
+end

data/lib/coil/queue_locking.rb

@@ -0,0 +1,108 @@
+# typed: true
+# frozen_string_literal: true
+
+module Coil
+  module QueueLocking
+    include Kernel # provides `loop` and `raise`
+    extend self
+
+    class Key
+      PREFIX = "QueueLocking"
+
+      # Construct a key suitable for obtaining a PostgreSQL advisory lock.
+      # https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS
+      def initialize(queue_type:, message_type:, message_key:)
+        @text = [PREFIX, queue_type, message_type, stringify(message_key)].join("|")
+        @int64 = Digest::SHA256.digest(@text).slice(0, 8).unpack1("q")
+      end
+
+      attr_reader :text
+      attr_reader :int64
+
+      private
+
+      def stringify(message_key)
+        case message_key
+        when Hash, Array
+          JSON.generate(normalize(message_key))
+        else
+          message_key.to_s
+        end
+      end
+
+      def normalize(obj)
+        case obj
+        when Hash
+          obj.stringify_keys.sort.map { |k, v| [k, normalize(v)] }.to_h
+        when Array
+          obj.map { |x| normalize(x) }
+        else
+          obj
+        end
+      end
+    end
+
+    class LockWaitTimeout < StandardError
+    end
+
+    # Run an action while holding advisory locks on a list of keys for messages
+    # of a given type in a given queue.
+    #
+    # By default, this waits until the locks can be obtained. To instead
+    # raise a QueueLocking::LockWaitTimeout error if locks cannot be obtained
+    # immediately, specify `wait: false`.
+    def locking(queue_type:, message_type:, message_keys:, wait: true, &blk)
+      keys = message_keys.compact.map do |message_key|
+        Key.new(queue_type:, message_type:, message_key:)
+      end
+
+      # Acquire locks in a consistent order to avoid deadlocks.
+      ks = keys.uniq(&:int64).sort_by(&:int64).reverse
+
+      fn = ks.reduce(blk) do |f, key|
+        -> do
+          with_lock(key:, wait:, &f)
+        end
+      end
+
+      ApplicationRecord.transaction(&fn)
+    end
+
+    private
+
+    def with_lock(key:, wait:, &blk)
+      wait ? lock(key:) : try_lock(key:)
+      blk.call
+    end
+
+    ACQUIRE_LOCK = "pg_advisory_xact_lock"
+    TRY_ACQUIRE_LOCK = "pg_try_advisory_xact_lock"
+
+    def lock(key:)
+      command = sql(fn: ACQUIRE_LOCK, key:)
+      connection.execute(command)
+    rescue ActiveRecord::LockWaitTimeout
+      raise LockWaitTimeout
+    end
+
+    def try_lock(key:)
+      query = sql(fn: TRY_ACQUIRE_LOCK, key:)
+      return if connection.select_value(query)
+      raise LockWaitTimeout
+    end
+
+    def sql(fn:, key:)
+      arg = key.int64
+      name = SecureRandom.hex # avoid cached results
+      comment = key.text.gsub(%r{(/\*)|(\*/)}, "--")
+
+      <<~SQL.squish
+        SELECT #{fn}(#{arg}) AS "#{name}" /* #{comment} */
+      SQL
+    end
+
+    def connection
+      ApplicationRecord.connection
+    end
+  end
+end

data/lib/coil/version.rb

@@ -0,0 +1,6 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Coil
+  VERSION = "1.3.0"
+end

data/lib/coil.rb

@@ -0,0 +1,9 @@
+# typed: strict
+
+require "rails"
+require "coil/version"
+require "coil/engine"
+require "coil/queue_locking"
+
+module Coil
+end

data/lib/tasks/coil_tasks.rake

@@ -0,0 +1,6 @@
+# typed: strict
+
+# desc "Explaining what the task does"
+# task :coil do
+#   # Task goes here
+# end