journaled 6.2.1 → 6.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2e688048716e5c43e64a60599615e6df3020065155b4acbffdbd3ac7341a460
-  data.tar.gz: 0c18dd4b468168d6d4d96c52e65f29ca7d480c0549cb41e668fd73be6149bda0
+  metadata.gz: 95198670d7f94155a1de312881e9f74d1ad0f80fe10b65080cf6db8f27332ab4
+  data.tar.gz: 43c8dc50365e0a68aed87cb4cb69a2632f23113e4c9f3e1cc71ec239659713df
 SHA512:
-  metadata.gz: 166663bbc460b1ab3bc88924bf5a28f7444ab2bdcde42ad1aee46cc9c5fa532a4623203c58588e5dd96ec7d9064c8cf511193b465e09943a666e754b23f4ba7c
-  data.tar.gz: 8ab6e8278746681bb4216a5afb6f6fee8e6337ad6399247ea702b27741ad838810bebcdc1ac17ab7e8470e56de73f9fe0a572ba5ea2004908a2f290762e71c18
+  metadata.gz: 5a06d677d4e1aaa042f2c57d4d6a3fd6d9e72c657d737b7dacff976caac0a08e6a48101a968dbb9c4a23f2f523a10cdf66383ce8b637193cbce2c0fd404fe0a2
+  data.tar.gz: fb7a06748b42787878ad50d126be1bc8607ae0e3140235cd4fc4505cfcb2e239cc612b97de323c7dec050ca25bc2fa365323dd4a07230642e59e75ee52012bd8

data/README.md

@@ -164,6 +164,25 @@ Journaling provides a number of different configuation options that can be set i
   Journaled.outbox_base_class_name = 'EventsRecord'
   ```
 
+#### `Journaled.outbox_processing_mode` (default: `:batch`)
+
+  **Only relevant when using `Journaled::Outbox::Adapter`.**
+
+  Controls how events are sent to Kinesis. Two modes are available:
+
+  - **`:batch`** (default) - Uses the Kinesis `put_records` batch API for high throughput. Events are sent in parallel batches, allowing multiple workers to run concurrently. Best for most use cases where strict ordering is not required.
+
+  - **`:guaranteed_order`** - Uses the Kinesis `put_record` single-event API to send events sequentially. Events are processed one at a time in order, stopping on the first transient failure to preserve ordering. Use this when you need strict ordering guarantees per partition key. Note: The current implementation requires single-threaded processing, but future optimizations may support batching and multi-threading by partition key.
+
+  Example:
+  ```ruby
+  # For high throughput (default)
+  Journaled.outbox_processing_mode = :batch
+
+  # For guaranteed ordering
+  Journaled.outbox_processing_mode = :guaranteed_order
+  ```
+
 #### ActiveJob `set` options
 
 Both model-level directives accept additional options to be passed into ActiveJob's `set` method:
@@ -182,6 +201,8 @@ journal_attributes :email, enqueue_with: { priority: 20, queue: 'journaled' }
 
 Journaled includes a built-in Outbox-style delivery adapter with horizontally scalable workers.
 
+By default, the Outbox adapter uses the Kinesis `put_records` batch API for high-throughput event processing, allowing multiple workers to process events in parallel. If you require strict ordering guarantees per partition key, you can configure sequential processing mode (see configuration options below).
+
 **Setup:**
 
 This feature requires creating database tables and is completely optional. Existing users are unaffected.
@@ -207,6 +228,16 @@ Journaled.delivery_adapter = Journaled::Outbox::Adapter
 # Optional: Customize worker behavior (these are the defaults)
 Journaled.worker_batch_size = 500        # Max events per Kinesis batch (Kinesis API limit)
 Journaled.worker_poll_interval = 5       # Seconds between polls
+
+# Optional: Configure processing mode (default: :batch)
+# - :batch - Uses Kinesis put_records batch API for high throughput (default)
+#            Events are sent in parallel batches. Multiple workers can run concurrently.
+# - :guaranteed_order - Uses Kinesis put_record single-event API for sequential processing
+#                       Events are sent one at a time in order. Use this if you need
+#                       strict ordering guarantees per partition key. The current
+#                       implementation processes events single-threaded, though future
+#                       optimizations may support batching/multi-threading by partition key.
+Journaled.outbox_processing_mode = :batch
 ```
 
 **Note:** When using the Outbox adapter, you do **not** need to configure an ActiveJob queue adapter (skip step 1 of Installation). The Outbox adapter uses the `journaled_outbox_events` table for event storage and its own worker daemons for processing, making it independent of ActiveJob. Transactional batching still works seamlessly with the Outbox adapter.
@@ -217,6 +248,8 @@ Journaled.worker_poll_interval = 5       # Seconds between polls
 bundle exec rake journaled_worker:work
 ```
 
+**Note:** In `:batch` mode (the default), you can run multiple worker processes concurrently for horizontal scaling. In `:guaranteed_order` mode, the current implementation is optimized for running a single worker to maintain ordering guarantees.
+
 4. **Monitoring:**
 
 The system emits `ActiveSupport::Notifications` events:

data/app/models/journaled/outbox/event.rb

@@ -29,12 +29,20 @@ module Journaled
 
       # Fetch a batch of events for processing using SELECT FOR UPDATE
       #
+      # In :guaranteed_order mode, uses blocking lock to ensure sequential processing.
+      # In :batch mode, uses SKIP LOCKED to allow parallel workers.
+      #
       # @return [Array<Journaled::Outbox::Event>] Events locked for processing
       def self.fetch_batch_for_update
-        ready_to_process
-          .limit(Journaled.worker_batch_size)
-          .lock
-          .to_a
+        query = ready_to_process.limit(Journaled.worker_batch_size)
+
+        lock_clause = if Journaled.outbox_processing_mode == :guaranteed_order
+          'FOR UPDATE'
+        else
+          'FOR UPDATE SKIP LOCKED'
+        end
+
+        query.lock(lock_clause).to_a
       end
 
       # Requeue a failed event for processing

data/lib/journaled/kinesis_batch_sender.rb

@@ -1,98 +1,108 @@
 # frozen_string_literal: true
 
 module Journaled
-  # Sends batches of events to Kinesis using the PutRecord single-event API
+  # Sends batches of events to Kinesis using the PutRecords batch API
   #
   # This class handles:
-  # - Sending events individually to support guaranteed ordering
+  # - Sending events in batches to improve throughput
   # - Handling failures on a per-event basis
   # - Classifying errors as transient vs permanent
   #
   # Returns structured results for the caller to handle event state management.
   class KinesisBatchSender
-    FailedEvent = Struct.new(:event, :error_code, :error_message, :transient, keyword_init: true) do
-      def transient?
-        transient
-      end
-
-      def permanent?
-        !transient
-      end
-    end
-
-    PERMANENT_ERROR_CLASSES = [
-      Aws::Kinesis::Errors::ValidationException,
+    # Per-record error codes that indicate permanent failures (bad event data)
+    PERMANENT_ERROR_CODES = [
+      'ValidationException',
     ].freeze
 
     # Send a batch of database events to Kinesis
     #
-    # Sends events one at a time to guarantee ordering. Stops on first transient failure.
+    # Uses put_records batch API. Groups events by stream and sends each group as a batch.
     #
     # @param events [Array<Journaled::Outbox::Event>] Events to send
     # @return [Hash] Result with:
     #   - succeeded: Array of successfully sent events
-    #   - failed: Array of FailedEvent structs (only permanent failures)
+    #   - failed: Array of FailedEvent structs (both transient and permanent failures)
     def send_batch(events)
-      result = { succeeded: [], failed: [] }
+      # Group events by stream since put_records requires all records to go to the same stream
+      events.group_by(&:stream_name).each_with_object({ succeeded: [], failed: [] }) do |(stream_name, stream_events), result|
+        batch_result = send_stream_batch(stream_name, stream_events)
+        result[:succeeded].concat(batch_result[:succeeded])
+        result[:failed].concat(batch_result[:failed])
+      end
+    end
 
-      events.each do |event|
-        event_result = send_event(event)
-        if event_result.is_a?(FailedEvent)
-          if event_result.transient?
-            emit_transient_failure_metric
-            break
-          else
-            result[:failed] << event_result
-          end
-        else
-          result[:succeeded] << event_result
-        end
+    private
+
+    def send_stream_batch(stream_name, stream_events)
+      records = build_records(stream_events)
+
+      begin
+        response = kinesis_client.put_records(stream_name:, records:)
+        process_response(response, stream_events)
+      rescue Aws::Kinesis::Errors::ValidationException
+        # Re-raise batch-level validation errors (configuration issues)
+        # These indicate invalid stream name, batch too large, etc.
+        # Not event data problems - requires manual intervention
+        raise
+      rescue StandardError => e
+        # Handle transient errors (throttling, network issues, service unavailable)
+        handle_transient_batch_error(e, stream_events)
       end
+    end
 
-      result
+    def build_records(stream_events)
+      stream_events.map do |event|
+        {
+          data: event.event_data.merge(id: event.id).to_json,
+          partition_key: event.partition_key,
+        }
+      end
     end
 
-    private
+    def process_response(response, stream_events)
+      succeeded = []
+      failed = []
 
-    # Send a single event to Kinesis
-    #
-    # @param event [Journaled::Outbox::Event] Event to send
-    # @return [Journaled::Outbox::Event, FailedEvent] The event on success, or FailedEvent on failure
-    def send_event(event)
-      # Merge the DB-generated ID into the event data before sending to Kinesis
-      event_data_with_id = event.event_data.merge(id: event.id)
+      response.records.each_with_index do |record_result, index|
+        event = stream_events[index]
 
-      kinesis_client.put_record(
-        stream_name: event.stream_name,
-        data: event_data_with_id.to_json,
-        partition_key: event.partition_key,
-      )
+        if record_result.error_code
+          failed << create_failed_event(event, record_result)
+        else
+          succeeded << event
+        end
+      end
 
-      event
-    rescue *PERMANENT_ERROR_CLASSES => e
-      Rails.logger.error("Kinesis event send failed (permanent): #{e.class} - #{e.message}")
-      FailedEvent.new(
-        event:,
-        error_code: e.class.to_s,
-        error_message: e.message,
-        transient: false,
-      )
-    rescue StandardError => e
-      Rails.logger.error("Kinesis event send failed (transient): #{e.class} - #{e.message}")
-      FailedEvent.new(
+      { succeeded:, failed: }
+    end
+
+    def create_failed_event(event, record_result)
+      Journaled::KinesisFailedEvent.new(
         event:,
-        error_code: e.class.to_s,
-        error_message: e.message,
-        transient: true,
+        error_code: record_result.error_code,
+        error_message: record_result.error_message,
+        transient: PERMANENT_ERROR_CODES.exclude?(record_result.error_code),
       )
     end
 
-    def kinesis_client
-      @kinesis_client ||= KinesisClientFactory.build
+    def handle_transient_batch_error(error, stream_events)
+      Rails.logger.error("Kinesis batch send failed (transient): #{error.class} - #{error.message}")
+
+      failed = stream_events.map do |event|
+        Journaled::KinesisFailedEvent.new(
+          event:,
+          error_code: error.class.to_s,
+          error_message: error.message,
+          transient: true,
+        )
+      end
+
+      { succeeded: [], failed: }
     end
 
-    def emit_transient_failure_metric
-      ActiveSupport::Notifications.instrument('journaled.kinesis_batch_sender.transient_failure')
+    def kinesis_client
+      @kinesis_client ||= KinesisClientFactory.build
     end
   end
 end

data/lib/journaled/kinesis_failed_event.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Journaled
+  # Represents a failed event from Kinesis send operations
+  #
+  # Used by both KinesisBatchSender and KinesisSequentialSender to represent
+  # events that failed to send to Kinesis, along with error details and whether
+  # the failure is transient (retriable) or permanent.
+  KinesisFailedEvent = Struct.new(:event, :error_code, :error_message, :transient, keyword_init: true) do
+    def transient?
+      transient
+    end
+
+    def permanent?
+      !transient
+    end
+  end
+end

data/lib/journaled/kinesis_sequential_sender.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Journaled
+  # Sends batches of events to Kinesis using the PutRecord single-event API
+  #
+  # This class handles:
+  # - Sending events individually in order to support guaranteed ordering
+  # - Stopping on first transient failure to preserve ordering
+  # - Classifying errors as transient vs permanent
+  #
+  # Returns structured results for the caller to handle event state management.
+  class KinesisSequentialSender
+    PERMANENT_ERROR_CLASSES = [
+      Aws::Kinesis::Errors::ValidationException,
+    ].freeze
+
+    # Send a batch of database events to Kinesis
+    #
+    # Sends events one at a time to guarantee ordering. Stops on first transient failure.
+    #
+    # @param events [Array<Journaled::Outbox::Event>] Events to send
+    # @return [Hash] Result with:
+    #   - succeeded: Array of successfully sent events
+    #   - failed: Array of FailedEvent structs (only permanent failures)
+    def send_batch(events)
+      result = { succeeded: [], failed: [] }
+
+      events.each do |event|
+        event_result = send_event(event)
+        if event_result.is_a?(Journaled::KinesisFailedEvent)
+          if event_result.transient?
+            emit_transient_failure_metric
+            break
+          else
+            result[:failed] << event_result
+          end
+        else
+          result[:succeeded] << event_result
+        end
+      end
+
+      result
+    end
+
+    private
+
+    # Send a single event to Kinesis
+    #
+    # @param event [Journaled::Outbox::Event] Event to send
+    # @return [Journaled::Outbox::Event, FailedEvent] The event on success, or FailedEvent on failure
+    def send_event(event)
+      kinesis_client.put_record(
+        stream_name: event.stream_name,
+        data: event.event_data.merge(id: event.id).to_json,
+        partition_key: event.partition_key,
+      )
+
+      event
+    rescue *PERMANENT_ERROR_CLASSES => e
+      Rails.logger.error("[Journaled] Kinesis event send failed (permanent): #{e.class} - #{e.message}")
+      Journaled::KinesisFailedEvent.new(
+        event:,
+        error_code: e.class.to_s,
+        error_message: e.message,
+        transient: false,
+      )
+    rescue StandardError => e
+      Rails.logger.error("[Journaled] Kinesis event send failed (transient): #{e.class} - #{e.message}")
+      Journaled::KinesisFailedEvent.new(
+        event:,
+        error_code: e.class.to_s,
+        error_message: e.message,
+        transient: true,
+      )
+    end
+
+    def kinesis_client
+      @kinesis_client ||= KinesisClientFactory.build
+    end
+
+    def emit_transient_failure_metric
+      ActiveSupport::Notifications.instrument('journaled.kinesis_sequential_sender.transient_failure')
+    end
+  end
+end

data/lib/journaled/outbox/batch_processor.rb

@@ -6,31 +6,36 @@ module Journaled
     #
     # This class handles the core business logic of:
     # - Fetching events from the database (with FOR UPDATE)
-    # - Sending them to Kinesis one at a time to guarantee ordering
+    # - Sending them to Kinesis (batch API or sequential)
     # - Handling successful deliveries (deleting events)
     # - Handling permanent failures (marking with failed_at)
-    # - Handling ephemeral failures (stopping processing and committing)
+    # - Handling transient failures (leaving unlocked for retry)
     #
-    # Events are processed one at a time to guarantee ordering. If an event fails
-    # with an ephemeral error, processing stops and the transaction commits
-    # (deleting successes and marking permanent failures), then the loop re-enters.
+    # Supports two modes based on Journaled.outbox_processing_mode:
+    # - :batch - Uses put_records API for high throughput with parallel workers
+    # - :guaranteed_order - Uses put_record API for sequential processing
     #
     # All operations happen within a single database transaction for consistency.
     # The Worker class delegates to this for actual event processing.
     class BatchProcessor
       def initialize
-        @batch_sender = KinesisBatchSender.new
+        @batch_sender = if Journaled.outbox_processing_mode == :guaranteed_order
+          KinesisSequentialSender.new
+        else
+          KinesisBatchSender.new
+        end
       end
 
       # Process a single batch of events
       #
       # Wraps the entire batch processing in a single transaction:
       # 1. SELECT FOR UPDATE (claim events)
-      # 2. Send to Kinesis (batch sender handles one-at-a-time and short-circuiting)
+      # 2. Send to Kinesis (batch API or sequential, based on mode)
       # 3. Delete successful events
-      # 4. Mark failed events (batch sender only returns permanent failures)
+      # 4. Mark permanently failed events
+      # 5. Leave transient failures untouched (will be retried)
       #
-      # @return [Hash] Statistics with :succeeded, :failed_permanently counts
+      # @return [Hash] Statistics with :succeeded, :failed_permanently, :failed_transiently counts
       def process_batch
         ActiveRecord::Base.transaction do
           events = Event.fetch_batch_for_update
@@ -38,20 +43,24 @@ module Journaled
 
           result = batch_sender.send_batch(events)
 
-          # Delete successful events
           Event.where(id: result[:succeeded].map(&:id)).delete_all if result[:succeeded].any?
 
-          # Mark failed events
-          mark_events_as_failed(result[:failed]) if result[:failed].any?
+          permanent_failures = result[:failed].select(&:permanent?)
+          transient_failures = result[:failed].select(&:transient?)
+
+          mark_events_as_failed(permanent_failures) if permanent_failures.any?
 
           Rails.logger.info(
             "[journaled] Batch complete: #{result[:succeeded].count} succeeded, " \
-            "#{result[:failed].count} marked as failed (batch size: #{events.count})",
+            "#{permanent_failures.count} permanently failed, " \
+            "#{transient_failures.count} transiently failed (will retry) " \
+            "(batch size: #{events.count})",
           )
 
           {
             succeeded: result[:succeeded].count,
-            failed_permanently: result[:failed].count,
+            failed_permanently: permanent_failures.count,
+            failed_transiently: transient_failures.count,
           }
         end
       end

data/lib/journaled/outbox/metric_emitter.rb

@@ -12,13 +12,14 @@ module Journaled
 
       # Emit batch processing metrics
       #
-      # @param stats [Hash] Processing statistics with :succeeded, :failed_permanently
+      # @param stats [Hash] Processing statistics with :succeeded, :failed_permanently, :failed_transiently
       def emit_batch_metrics(stats)
-        total_events = stats[:succeeded] + stats[:failed_permanently]
+        total_events = stats[:succeeded] + stats[:failed_permanently] + stats[:failed_transiently]
 
         emit_metric('journaled.worker.batch_process', value: total_events)
         emit_metric('journaled.worker.batch_sent', value: stats[:succeeded])
-        emit_metric('journaled.worker.batch_failed', value: stats[:failed_permanently])
+        emit_metric('journaled.worker.batch_failed_permanently', value: stats[:failed_permanently])
+        emit_metric('journaled.worker.batch_failed_transiently', value: stats[:failed_transiently])
       end
 
       # Collect and emit queue metrics

data/lib/journaled/outbox/worker.rb

@@ -60,9 +60,8 @@ module Journaled
             break
           end
 
-          events_processed = 0
           begin
-            events_processed = process_batch
+            process_batch
             emit_metrics_if_needed
           rescue StandardError => e
             Rails.logger.error("Worker error: #{e.class} - #{e.message}")
@@ -71,20 +70,13 @@ module Journaled
 
           break if shutdown_requested
 
-          # Only sleep if no events were processed to prevent excessive polling on empty table
-          sleep(Journaled.worker_poll_interval) if events_processed.zero?
+          sleep(Journaled.worker_poll_interval)
         end
       end
 
       def process_batch
         stats = processor.process_batch
 
-        instrument_batch_results(stats)
-
-        stats[:succeeded] + stats[:failed_permanently]
-      end
-
-      def instrument_batch_results(stats)
         metric_emitter.emit_batch_metrics(stats)
       end
 

data/lib/journaled/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Journaled
-  VERSION = "6.2.1"
+  VERSION = "6.2.2"
 end

data/lib/journaled.rb

@@ -12,7 +12,9 @@ require 'journaled/delivery_adapter'
 require 'journaled/delivery_adapters/active_job_adapter'
 require 'journaled/outbox/adapter'
 require 'journaled/kinesis_client_factory'
+require 'journaled/kinesis_failed_event'
 require 'journaled/kinesis_batch_sender'
+require 'journaled/kinesis_sequential_sender'
 require 'journaled/outbox/batch_processor'
 require 'journaled/outbox/metric_emitter'
 require 'journaled/outbox/worker'
@@ -31,8 +33,9 @@ module Journaled
   mattr_writer(:transactional_batching_enabled) { true }
 
   # Worker configuration (for Outbox-style event processing)
-  mattr_accessor(:worker_batch_size) { 1000 }
-  mattr_accessor(:worker_poll_interval) { 1 } # seconds
+  mattr_accessor(:worker_batch_size) { 500 }
+  mattr_accessor(:worker_poll_interval) { 0.5 } # seconds
+  mattr_accessor(:outbox_processing_mode) { :batch } # :batch or :guaranteed_order
 
   def self.transactional_batching_enabled?
     Thread.current[:journaled_transactional_batching_enabled] || @@transactional_batching_enabled

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: journaled
 version: !ruby/object:Gem::Version
-  version: 6.2.1
+  version: 6.2.2
 platform: ruby
 authors:
 - Jake Lipson
@@ -274,6 +274,8 @@ files:
 - lib/journaled/errors.rb
 - lib/journaled/kinesis_batch_sender.rb
 - lib/journaled/kinesis_client_factory.rb
+- lib/journaled/kinesis_failed_event.rb
+- lib/journaled/kinesis_sequential_sender.rb
 - lib/journaled/outbox/adapter.rb
 - lib/journaled/outbox/batch_processor.rb
 - lib/journaled/outbox/metric_emitter.rb