artery 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e07fd862261dcd6ea03bcdaeaa8ba1105f4acbdbd19a7be3c1a3a475955964a7
-  data.tar.gz: bad3ae0d8a661dde23887d06c52ef073f24b43f3774562525094430eb5882a55
+  metadata.gz: d38d9b951e14faf5d0a8950f6e1d23dd7d20e9c0ec0599daceaefcbbf5279a86
+  data.tar.gz: 624da77476449143e5298b6da48d39a1e7e0502efa574c146dd6ab5f9857c0ea
 SHA512:
-  metadata.gz: e6e7f7bd693b96dc421b6673ea818515546415c7a016364aff407de07b18e4b76d6a10dd59a34ea9f29b9f5cfddc39ad81fe1f816fa166c2bc135fd8105bf225
-  data.tar.gz: 6515ae08facc636e0334b6bddf24d37c4dbf8136d0f90db342771f346c68f302fa114368a707becc2da4e2b868369e05d6db5006971c12e00ad6e34372240b87
+  metadata.gz: afbb2b2cd62233852531faa4bc2a45487e1f78a719f939b23aec251592578dd4b451e7520f8491ac697bddd25a1d53537abca7759d6d6320863038b56b572272
+  data.tar.gz: 331626ae3d7bc48b488875918e7f7bc6b16f986e9238397a35a8acefd6f54d6347ca32e2f59f263b8562e8d5f07b864e171756f6146599816801fddbd6b2c136

data/README.md

@@ -27,6 +27,32 @@ $ rake artery:install:migrations
 $ rake db:migrate
 ```
 
+## Publishing modes
+
+Artery supports two modes for publishing messages to NATS:
+
+### Inline mode (`inline_publish = true`, default)
+
+Messages are published directly from the `after_commit` callback. No additional processes required -- convenient for development and testing. However, under high concurrency `after_commit` callbacks can execute out of order across transactions, which may lead to incorrect `_previous_index` values. **Recommended for development only.**
+
+### Publisher mode (`inline_publish = false`)
+
+A separate Publisher process polls `artery_messages` and publishes them to NATS in strict `id` order, guaranteeing a correct `_previous_index` chain. Messages are persisted inside the model transaction (`before_commit`) without any locks, so there is no contention overhead. **Recommended for production.**
+
+```ruby
+Artery.configure do |config|
+  config.inline_publish = false
+end
+```
+
+Running the publisher:
+
+```bash
+$ bundle exec artery-publisher
+```
+
+The publisher uses a `concurrent-ruby` thread pool. Pool size is controlled by `RAILS_MAX_THREADS` (default 5). Each model gets its own thread that polls for unpublished messages.
+
 ## Admin interface
 
 In admin interface you can list your artery endpoints and check their statuses.
@@ -46,6 +72,11 @@ Artery uses `ActiveSupport::Notifications` for instrumentation and `ActiveSuppor
 Artery.configure do |config|
   config.service_name = :my_service
 
+  # When true, messages are published inline from after_commit (no publisher needed).
+  # Set to false in production when running the publisher process.
+  # Default: true
+  config.inline_publish = false
+
   # Log every message (publish/request/subscribe/response).
   # When false, only lifecycle events (errors, sync, connect/disconnect) are logged.
   # Default: true
@@ -111,6 +142,10 @@ Available events (each uses a `stage:`, `state:`, or `action:` payload key to di
 | | | `:subscribing` | `route` | Subscribing to route |
 | `lock.artery` | `state` | `:waiting` | `latest_index` | Waiting for subscription lock |
 | | | `:acquired` | `latest_index` | Lock acquired |
+| `publisher.artery` | `action` | `:started` | — | Publisher loop started |
+| | | `:model_started` | `model` | Model polling thread started |
+| | | `:publishing` | `model`, `count` | Batch published (block, has duration) |
+| | | `:error` | `model`, `error` | Publisher error for model |
 
 ## Contributing
 Contribution directions go here.

data/app/models/concerns/artery/message_model.rb

@@ -44,9 +44,7 @@ module Artery
       raise NotImplementedError
     end
 
-    protected
-
-    def send_to_artery
+    def publish_to_artery(previous_index: self.previous_index)
       Artery.publish route, to_artery.merge('_previous_index' => previous_index)
     end
   end

data/db/migrate/20260326132618_add_last_published_id_to_artery_model_infos.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddLastPublishedIdToArteryModelInfos < ActiveRecord::Migration[5.2]
+  def change
+    add_column :artery_model_infos, :last_published_id, :bigint, null: false, default: 0
+  end
+end

data/exe/artery-publisher

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require ENV['APP_PATH'] || File.join(File.expand_path('.'), 'config', 'application')
+
+Rails.application.initialize!
+Rails.application.eager_load!
+
+Artery::Publisher.new.run

data/lib/artery/active_record/message.rb

@@ -9,10 +9,7 @@ module Artery
 
       serialize :data, coder: JSON
 
-      after_commit :send_to_artery, on: :create
-      around_create :lock_on_model
-
-      attr_accessor :cached_previous_index
+      after_commit :publish_to_artery, on: :create, if: -> { Artery.inline_publish? }
 
       alias index id
 
@@ -23,7 +20,7 @@ module Artery
         end
 
         def latest_index(model)
-          Artery.model_info_class.find_by(model: model)&.latest_index.to_i
+          where(model: model).maximum(:id).to_i
         end
 
         def delete_old
@@ -33,22 +30,11 @@ module Artery
       end
 
       def previous_index
-        return cached_previous_index if cached_previous_index
-
         scope = self.class.where(model: model).order(:id)
         scope = scope.where(self.class.arel_table[:id].lt(index)) if index
 
         scope.select(:id).last&.id.to_i
       end
-
-      private
-
-      def lock_on_model
-        lock_row = Artery.model_info_class.acquire_lock!(model)
-        self.cached_previous_index = lock_row.latest_index
-        yield
-        lock_row.update!(latest_index: id)
-      end
     end
   end
 end

data/lib/artery/active_record/model_info.rb

@@ -5,6 +5,10 @@ module Artery
     class ModelInfo < ::ActiveRecord::Base
       self.table_name = 'artery_model_infos'
 
+      # @deprecated The +latest_index+ column is no longer maintained.
+      #   Use {Artery::ActiveRecord::Message.latest_index} instead.
+
+      # @deprecated No longer used. Publishing is handled by {Artery::Publisher}.
       def self.acquire_lock!(model_name)
         lock_row = lock('FOR UPDATE').find_by(model: model_name)
         return lock_row if lock_row
@@ -17,6 +21,16 @@ module Artery
 
         lock('FOR UPDATE').find_by!(model: model_name)
       end
+
+      def self.ensure_initialized!(model_name)
+        row = find_or_create_by!(model: model_name) do |r|
+          r.latest_index = Artery.message_class.latest_index(model_name)
+        end
+
+        row.update!(last_published_id: row.latest_index) if row.last_published_id.zero? && row.latest_index.positive?
+
+        row
+      end
     end
   end
 end

data/lib/artery/config.rb

@@ -39,7 +39,7 @@ module Artery
           @request_timeout || ENV.fetch('ARTERY_REQUEST_TIMEOUT', '15').to_i
         end
 
-        attr_writer :log_messages, :message_body_max_size
+        attr_writer :log_messages, :message_body_max_size, :inline_publish
 
         def log_messages?
           instance_variable_defined?(:@log_messages) ? @log_messages : true
@@ -49,6 +49,10 @@ module Artery
           instance_variable_defined?(:@message_body_max_size) ? @message_body_max_size : nil
         end
 
+        def inline_publish?
+          instance_variable_defined?(:@inline_publish) ? @inline_publish : true
+        end
+
         def error_handler
           @error_handler || (defined?(Artery::SentryErrorHandler) ? Artery::SentryErrorHandler : Artery::ErrorHandler)
         end

data/lib/artery/log_subscriber.rb

@@ -66,6 +66,17 @@ module Artery
       end
     end
 
+    def publisher(event)
+      p = event.payload
+
+      case p[:action]
+      when :started then info 'started'
+      when :model_started then info 'polling started'
+      when :publishing then debug "published #{p[:count]} messages (#{event.duration.round(1)}ms)"
+      when :error then error p[:error]
+      end
+    end
+
     private
 
     def debug(msg)

data/lib/artery/model/callbacks.rb

@@ -14,57 +14,28 @@ module Artery
           after_archive   :artery_on_archive
           after_unarchive :artery_on_unarchive
         end
-
-        if artery[:non_atomic_notification]
-          after_commit :artery_send_pending_notifications
-        else
-          before_commit :artery_send_pending_notifications
-        end
       end
 
+      private
+
       def artery_on_create
-        artery_pending_notifications << [:create]
+        artery_notify_message(:create)
       end
 
       def artery_on_update
-        artery_pending_notifications << [:update]
+        artery_notify_message(:update)
       end
 
       def artery_on_archive
-        artery_pending_notifications << [:archive, { archived_at: archived_at.to_f }]
+        artery_notify_message(:archive, archived_at: archived_at.to_f)
       end
 
       def artery_on_unarchive
-        artery_pending_notifications << [:unarchive]
+        artery_notify_message(:unarchive)
       end
 
       def artery_on_destroy
-        artery_pending_notifications << [:delete]
-      end
-
-      private
-
-      def artery_pending_notifications
-        @artery_pending_notifications ||= []
-      end
-
-      def artery_send_pending_notifications
-        attempts = 0
-        begin
-          artery_pending_notifications.each do |action, extra_data|
-            artery_notify_message(action, extra_data || {})
-          end
-        rescue StandardError => e
-          attempts += 1
-          retry if self.class.artery[:non_atomic_notification] && attempts <= 3
-
-          Artery.handle_error Artery::Error.new(
-            "Failed to send artery notifications after #{attempts} attempts: #{e.message}",
-            original_exception: e
-          )
-        ensure
-          @artery_pending_notifications = nil
-        end
+        artery_notify_message(:delete)
       end
     end
   end

data/lib/artery/publisher.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module Artery
+  class Publisher
+    DISCOVERY_INTERVAL = 5
+    POLL_INTERVAL = 0.5
+    BATCH_SIZE = 100
+
+    def run
+      Artery.handle_signals { shutdown }
+      Artery.start { publisher_loop }
+    end
+
+    def shutdown
+      @pool&.shutdown
+      @pool&.wait_for_termination(10)
+    end
+
+    private
+
+    def publisher_loop
+      @pool = Concurrent::ThreadPoolExecutor.new(
+        min_threads: 1,
+        max_threads: ENV.fetch('RAILS_MAX_THREADS', 5).to_i,
+        max_queue: 0,
+        fallback_policy: :caller_runs
+      )
+      @running_models = Concurrent::Set.new
+
+      Instrumentation.instrument(:publisher, action: :started)
+
+      loop do
+        models = Artery.model_info_class.pluck(:model)
+
+        models.each do |model|
+          next if @running_models.include?(model)
+
+          @running_models.add(model)
+          @pool.post { model_loop(model) }
+        end
+
+        sleep DISCOVERY_INTERVAL
+      end
+    end
+
+    def model_loop(model)
+      Artery.logger.tagged('Publisher', model) do
+        Artery.model_info_class.ensure_initialized!(model)
+        Instrumentation.instrument(:publisher, action: :model_started, model: model)
+
+        loop do
+          published = publish_batch(model)
+          sleep POLL_INTERVAL if published < BATCH_SIZE
+        end
+      rescue StandardError => e
+        Instrumentation.instrument(:publisher, action: :error, model: model, error: e.message)
+        Artery.handle_error Error.new(
+          "Publisher error for #{model}: #{e.message}",
+          original_exception: e
+        )
+        sleep POLL_INTERVAL
+        retry
+      end
+    ensure
+      @running_models.delete(model)
+    end
+
+    def publish_batch(model)
+      Artery.model_info_class.transaction do
+        row = Artery.model_info_class.lock('FOR UPDATE').find_by!(model: model)
+
+        scope = Artery.message_class.where(model: model)
+        messages = scope.where(scope.arel_table[:id].gt(row.last_published_id))
+                        .order(:id)
+                        .limit(BATCH_SIZE)
+
+        return 0 if messages.empty?
+
+        prev_index = row.last_published_id
+        Instrumentation.instrument(:publisher, action: :publishing, model: model, count: messages.size) do
+          messages.each do |msg|
+            msg.publish_to_artery(previous_index: prev_index)
+            prev_index = msg.id
+          end
+        end
+
+        row.update!(last_published_id: prev_index)
+        messages.size
+      end
+    end
+  end
+end

data/lib/artery/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Artery
-  VERSION = '1.3.1'
+  VERSION = '1.4.0'
 end

data/lib/artery.rb

@@ -12,6 +12,7 @@ require_relative 'multiblock_has_block'
 
 module Artery
   autoload :Config,        'artery/config'
+  autoload :Publisher,     'artery/publisher'
   autoload :Worker,        'artery/worker'
   autoload :Sync,          'artery/sync'
   autoload :Check,         'artery/check'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: artery
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Sergey Gnuskov
@@ -37,6 +37,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: nats-pure
   requirement: !ruby/object:Gem::Requirement
@@ -82,6 +96,7 @@ email:
 executables:
 - artery-check
 - artery-clean
+- artery-publisher
 - artery-sync
 - artery-worker
 extensions: []
@@ -101,8 +116,10 @@ files:
 - db/migrate/20200109120305_remove_last_message_at_from_artery_subscription_infos.rb
 - db/migrate/20240411120304_add_synchronization_heartbeat_to_artery_subscription_infos.rb
 - db/migrate/20250320120000_create_artery_model_infos.rb
+- db/migrate/20260326132618_add_last_published_id_to_artery_model_infos.rb
 - exe/artery-check
 - exe/artery-clean
+- exe/artery-publisher
 - exe/artery-sync
 - exe/artery-worker
 - lib/artery.rb
@@ -127,6 +144,7 @@ files:
 - lib/artery/no_brainer.rb
 - lib/artery/no_brainer/message.rb
 - lib/artery/no_brainer/subscription_info.rb
+- lib/artery/publisher.rb
 - lib/artery/routing.rb
 - lib/artery/subscription.rb
 - lib/artery/subscription/incoming_message.rb