journaled 6.2.1 → 6.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2e688048716e5c43e64a60599615e6df3020065155b4acbffdbd3ac7341a460
-  data.tar.gz: 0c18dd4b468168d6d4d96c52e65f29ca7d480c0549cb41e668fd73be6149bda0
+  metadata.gz: ed85996fe76daec652ed49ec5c128e27f906c9ce91193de1b394843e11ed3971
+  data.tar.gz: '09335faed12c2732e425535b849f36d7973a2a614ffd52bd1a77ac0e5b251ba5'
 SHA512:
-  metadata.gz: 166663bbc460b1ab3bc88924bf5a28f7444ab2bdcde42ad1aee46cc9c5fa532a4623203c58588e5dd96ec7d9064c8cf511193b465e09943a666e754b23f4ba7c
-  data.tar.gz: 8ab6e8278746681bb4216a5afb6f6fee8e6337ad6399247ea702b27741ad838810bebcdc1ac17ab7e8470e56de73f9fe0a572ba5ea2004908a2f290762e71c18
+  metadata.gz: e0f867425b1f9a033644b007b97f144665d1bf5c30f498621ad2a805d3d607158a219fe46e3ef189389a89b77c038289dc45943e659519c9932b1bcc5832b1fa
+  data.tar.gz: 1788bf801f0c3a3d2dafb5108532e350dce725319312367351fd0b9505cc554ce7a03d85c15b36a2c2cddbef313818d3770beeee9aaf1c8550e812925830fdc9

data/README.md

@@ -164,6 +164,25 @@ Journaling provides a number of different configuation options that can be set i
   Journaled.outbox_base_class_name = 'EventsRecord'
   ```
 
+#### `Journaled.outbox_processing_mode` (default: `:batch`)
+
+  **Only relevant when using `Journaled::Outbox::Adapter`.**
+
+  Controls how events are sent to Kinesis. Two modes are available:
+
+  - **`:batch`** (default) - Uses the Kinesis `put_records` batch API for high throughput. Events are sent in parallel batches, allowing multiple workers to run concurrently. Best for most use cases where strict ordering is not required.
+
+  - **`:guaranteed_order`** - Uses the Kinesis `put_record` single-event API to send events sequentially. Events are processed one at a time in order, stopping on the first transient failure to preserve ordering. Use this when you need strict ordering guarantees per partition key. Note: The current implementation requires single-threaded processing, but future optimizations may support batching and multi-threading by partition key.
+
+  Example:
+  ```ruby
+  # For high throughput (default)
+  Journaled.outbox_processing_mode = :batch
+
+  # For guaranteed ordering
+  Journaled.outbox_processing_mode = :guaranteed_order
+  ```
+
 #### ActiveJob `set` options
 
 Both model-level directives accept additional options to be passed into ActiveJob's `set` method:
@@ -182,6 +201,8 @@ journal_attributes :email, enqueue_with: { priority: 20, queue: 'journaled' }
 
 Journaled includes a built-in Outbox-style delivery adapter with horizontally scalable workers.
 
+By default, the Outbox adapter uses the Kinesis `put_records` batch API for high-throughput event processing, allowing multiple workers to process events in parallel. If you require strict ordering guarantees per partition key, you can configure sequential processing mode (see configuration options below).
+
 **Setup:**
 
 This feature requires creating database tables and is completely optional. Existing users are unaffected.
@@ -207,6 +228,16 @@ Journaled.delivery_adapter = Journaled::Outbox::Adapter
 # Optional: Customize worker behavior (these are the defaults)
 Journaled.worker_batch_size = 500        # Max events per Kinesis batch (Kinesis API limit)
 Journaled.worker_poll_interval = 5       # Seconds between polls
+
+# Optional: Configure processing mode (default: :batch)
+# - :batch - Uses Kinesis put_records batch API for high throughput (default)
+#            Events are sent in parallel batches. Multiple workers can run concurrently.
+# - :guaranteed_order - Uses Kinesis put_record single-event API for sequential processing
+#                       Events are sent one at a time in order. Use this if you need
+#                       strict ordering guarantees per partition key. The current
+#                       implementation processes events single-threaded, though future
+#                       optimizations may support batching/multi-threading by partition key.
+Journaled.outbox_processing_mode = :batch
 ```
 
 **Note:** When using the Outbox adapter, you do **not** need to configure an ActiveJob queue adapter (skip step 1 of Installation). The Outbox adapter uses the `journaled_outbox_events` table for event storage and its own worker daemons for processing, making it independent of ActiveJob. Transactional batching still works seamlessly with the Outbox adapter.
@@ -217,6 +248,8 @@ Journaled.worker_poll_interval = 5       # Seconds between polls
 bundle exec rake journaled_worker:work
 ```
 
+**Note:** In `:batch` mode (the default), you can run multiple worker processes concurrently for horizontal scaling. In `:guaranteed_order` mode, the current implementation is optimized for running a single worker to maintain ordering guarantees.
+
 4. **Monitoring:**
 
 The system emits `ActiveSupport::Notifications` events:

data/app/models/journaled/outbox/event.rb

@@ -29,12 +29,20 @@ module Journaled
 
       # Fetch a batch of events for processing using SELECT FOR UPDATE
       #
+      # In :guaranteed_order mode, uses blocking lock to ensure sequential processing.
+      # In :batch mode, uses SKIP LOCKED to allow parallel workers.
+      #
       # @return [Array<Journaled::Outbox::Event>] Events locked for processing
       def self.fetch_batch_for_update
-        ready_to_process
-          .limit(Journaled.worker_batch_size)
-          .lock
-          .to_a
+        query = ready_to_process.limit(Journaled.worker_batch_size)
+
+        lock_clause = if Journaled.outbox_processing_mode == :guaranteed_order
+          'FOR UPDATE'
+        else
+          'FOR UPDATE SKIP LOCKED'
+        end
+
+        query.lock(lock_clause).to_a
       end
 
       # Requeue a failed event for processing

data/lib/journaled/kinesis_batch_sender.rb

@@ -1,98 +1,115 @@
 # frozen_string_literal: true
 
 module Journaled
-  # Sends batches of events to Kinesis using the PutRecord single-event API
+  # Sends batches of events to Kinesis using the PutRecords batch API
   #
   # This class handles:
-  # - Sending events individually to support guaranteed ordering
+  # - Sending events in batches to improve throughput
   # - Handling failures on a per-event basis
   # - Classifying errors as transient vs permanent
   #
   # Returns structured results for the caller to handle event state management.
   class KinesisBatchSender
-    FailedEvent = Struct.new(:event, :error_code, :error_message, :transient, keyword_init: true) do
-      def transient?
-        transient
-      end
-
-      def permanent?
-        !transient
-      end
-    end
-
-    PERMANENT_ERROR_CLASSES = [
-      Aws::Kinesis::Errors::ValidationException,
+    # Per-record error codes that indicate permanent failures (bad event data)
+    PERMANENT_ERROR_CODES = [
+      'ValidationException',
     ].freeze
 
     # Send a batch of database events to Kinesis
     #
-    # Sends events one at a time to guarantee ordering. Stops on first transient failure.
+    # Uses put_records batch API. Groups events by stream and sends each group as a batch.
     #
     # @param events [Array<Journaled::Outbox::Event>] Events to send
     # @return [Hash] Result with:
     #   - succeeded: Array of successfully sent events
-    #   - failed: Array of FailedEvent structs (only permanent failures)
+    #   - failed: Array of FailedEvent structs (both transient and permanent failures)
     def send_batch(events)
-      result = { succeeded: [], failed: [] }
-
-      events.each do |event|
-        event_result = send_event(event)
-        if event_result.is_a?(FailedEvent)
-          if event_result.transient?
-            emit_transient_failure_metric
-            break
-          else
-            result[:failed] << event_result
-          end
+      # Group events by stream since put_records requires all records to go to the same stream
+      events.group_by(&:stream_name).each_with_object({ succeeded: [], failed: [] }) do |(stream_name, stream_events), result|
+        batch_result = send_stream_batch(stream_name, stream_events)
+        result[:succeeded].concat(batch_result[:succeeded])
+        result[:failed].concat(batch_result[:failed])
+      end
+    end
+
+    private
+
+    def send_stream_batch(stream_name, stream_events)
+      records = build_records(stream_events)
+
+      begin
+        response = kinesis_client.put_records(stream_name:, records:)
+        process_response(response, stream_events)
+      rescue Aws::Kinesis::Errors::ValidationException
+        # Re-raise batch-level validation errors (configuration issues)
+        # These indicate invalid stream name, batch too large, etc.
+        # Not event data problems - requires manual intervention
+        raise
+      rescue StandardError => e
+        # Handle transient errors (throttling, network issues, service unavailable)
+        handle_transient_batch_error(e, stream_events)
+      end
+    end
+
+    def build_records(stream_events)
+      stream_events.map do |event|
+        {
+          data: event.event_data.merge(id: event.id).to_json,
+          partition_key: event.partition_key,
+        }
+      end
+    end
+
+    def process_response(response, stream_events)
+      succeeded = []
+      failed = []
+
+      response.records.each_with_index do |record_result, index|
+        event = stream_events[index]
+
+        if record_result.error_code
+          failed << create_failed_event(
+            event,
+            error_code: record_result.error_code,
+            error_message: record_result.error_message,
+            transient: PERMANENT_ERROR_CODES.exclude?(record_result.error_code),
+          )
         else
-          result[:succeeded] << event_result
+          succeeded << event
         end
       end
 
-      result
+      { succeeded:, failed: }
     end
 
-    private
-
-    # Send a single event to Kinesis
-    #
-    # @param event [Journaled::Outbox::Event] Event to send
-    # @return [Journaled::Outbox::Event, FailedEvent] The event on success, or FailedEvent on failure
-    def send_event(event)
-      # Merge the DB-generated ID into the event data before sending to Kinesis
-      event_data_with_id = event.event_data.merge(id: event.id)
-
-      kinesis_client.put_record(
-        stream_name: event.stream_name,
-        data: event_data_with_id.to_json,
-        partition_key: event.partition_key,
-      )
+    def create_failed_event(event, error_code:, error_message:, transient:)
+      Outbox::MetricEmitter.emit_kinesis_failure(event:, error_code:)
 
-      event
-    rescue *PERMANENT_ERROR_CLASSES => e
-      Rails.logger.error("Kinesis event send failed (permanent): #{e.class} - #{e.message}")
-      FailedEvent.new(
-        event:,
-        error_code: e.class.to_s,
-        error_message: e.message,
-        transient: false,
-      )
-    rescue StandardError => e
-      Rails.logger.error("Kinesis event send failed (transient): #{e.class} - #{e.message}")
-      FailedEvent.new(
+      Journaled::KinesisFailedEvent.new(
         event:,
-        error_code: e.class.to_s,
-        error_message: e.message,
-        transient: true,
+        error_code:,
+        error_message:,
+        transient:,
       )
     end
 
-    def kinesis_client
-      @kinesis_client ||= KinesisClientFactory.build
+    def handle_transient_batch_error(error, stream_events)
+      Rails.logger.error("Kinesis batch send failed (transient): #{error.class} - #{error.message}")
+
+      failed = stream_events.map do |event|
+        create_failed_event(
+          event,
+          error_code: error.class.to_s,
+          error_message: error.message,
+          transient: true,
+        )
+      end
+
+      { succeeded: [], failed: }
     end
 
-    def emit_transient_failure_metric
-      ActiveSupport::Notifications.instrument('journaled.kinesis_batch_sender.transient_failure')
+    def kinesis_client
+      @kinesis_client ||= KinesisClientFactory.build
     end
   end
 end

data/lib/journaled/kinesis_failed_event.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Journaled
+  # Represents a failed event from Kinesis send operations
+  #
+  # Used by both KinesisBatchSender and KinesisSequentialSender to represent
+  # events that failed to send to Kinesis, along with error details and whether
+  # the failure is transient (retriable) or permanent.
+  KinesisFailedEvent = Struct.new(:event, :error_code, :error_message, :transient, keyword_init: true) do
+    def transient?
+      transient
+    end
+
+    def permanent?
+      !transient
+    end
+  end
+end

data/lib/journaled/kinesis_sequential_sender.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Journaled
+  # Sends batches of events to Kinesis using the PutRecord single-event API
+  #
+  # This class handles:
+  # - Sending events individually in order to support guaranteed ordering
+  # - Stopping on first transient failure to preserve ordering
+  # - Classifying errors as transient vs permanent
+  #
+  # Returns structured results for the caller to handle event state management.
+  class KinesisSequentialSender
+    PERMANENT_ERROR_CLASSES = [
+      Aws::Kinesis::Errors::ValidationException,
+    ].freeze
+
+    # Send a batch of database events to Kinesis
+    #
+    # Sends events one at a time to guarantee ordering. Stops on first transient failure.
+    #
+    # @param events [Array<Journaled::Outbox::Event>] Events to send
+    # @return [Hash] Result with:
+    #   - succeeded: Array of successfully sent events
+    #   - failed: Array of FailedEvent structs (only permanent failures)
+    def send_batch(events)
+      result = { succeeded: [], failed: [] }
+
+      events.each do |event|
+        event_result = send_event(event)
+        if event_result.is_a?(Journaled::KinesisFailedEvent)
+          if event_result.transient?
+            emit_transient_failure_metric
+            break
+          else
+            result[:failed] << event_result
+          end
+        else
+          result[:succeeded] << event_result
+        end
+      end
+
+      result
+    end
+
+    private
+
+    # Send a single event to Kinesis
+    #
+    # @param event [Journaled::Outbox::Event] Event to send
+    # @return [Journaled::Outbox::Event, FailedEvent] The event on success, or FailedEvent on failure
+    def send_event(event)
+      kinesis_client.put_record(
+        stream_name: event.stream_name,
+        data: event.event_data.merge(id: event.id).to_json,
+        partition_key: event.partition_key,
+      )
+
+      event
+    rescue *PERMANENT_ERROR_CLASSES => e
+      Rails.logger.error("[Journaled] Kinesis event send failed (permanent): #{e.class} - #{e.message}")
+      error_code = e.class.to_s
+      Outbox::MetricEmitter.emit_kinesis_failure(event:, error_code:)
+
+      Journaled::KinesisFailedEvent.new(
+        event:,
+        error_code:,
+        error_message: e.message,
+        transient: false,
+      )
+    rescue StandardError => e
+      Rails.logger.error("[Journaled] Kinesis event send failed (transient): #{e.class} - #{e.message}")
+      error_code = e.class.to_s
+      Outbox::MetricEmitter.emit_kinesis_failure(event:, error_code:)
+
+      Journaled::KinesisFailedEvent.new(
+        event:,
+        error_code:,
+        error_message: e.message,
+        transient: true,
+      )
+    end
+
+    def kinesis_client
+      @kinesis_client ||= KinesisClientFactory.build
+    end
+
+    def emit_transient_failure_metric
+      ActiveSupport::Notifications.instrument('journaled.kinesis_sequential_sender.transient_failure')
+    end
+  end
+end

data/lib/journaled/outbox/batch_processor.rb

@@ -6,31 +6,36 @@ module Journaled
     #
     # This class handles the core business logic of:
     # - Fetching events from the database (with FOR UPDATE)
-    # - Sending them to Kinesis one at a time to guarantee ordering
+    # - Sending them to Kinesis (batch API or sequential)
     # - Handling successful deliveries (deleting events)
     # - Handling permanent failures (marking with failed_at)
-    # - Handling ephemeral failures (stopping processing and committing)
+    # - Handling transient failures (leaving unlocked for retry)
     #
-    # Events are processed one at a time to guarantee ordering. If an event fails
-    # with an ephemeral error, processing stops and the transaction commits
-    # (deleting successes and marking permanent failures), then the loop re-enters.
+    # Supports two modes based on Journaled.outbox_processing_mode:
+    # - :batch - Uses put_records API for high throughput with parallel workers
+    # - :guaranteed_order - Uses put_record API for sequential processing
     #
     # All operations happen within a single database transaction for consistency.
     # The Worker class delegates to this for actual event processing.
     class BatchProcessor
       def initialize
-        @batch_sender = KinesisBatchSender.new
+        @batch_sender = if Journaled.outbox_processing_mode == :guaranteed_order
+          KinesisSequentialSender.new
+        else
+          KinesisBatchSender.new
+        end
       end
 
       # Process a single batch of events
       #
       # Wraps the entire batch processing in a single transaction:
       # 1. SELECT FOR UPDATE (claim events)
-      # 2. Send to Kinesis (batch sender handles one-at-a-time and short-circuiting)
+      # 2. Send to Kinesis (batch API or sequential, based on mode)
       # 3. Delete successful events
-      # 4. Mark failed events (batch sender only returns permanent failures)
+      # 4. Mark permanently failed events
+      # 5. Leave transient failures untouched (will be retried)
       #
-      # @return [Hash] Statistics with :succeeded, :failed_permanently counts
+      # @return [Hash] Statistics with :succeeded, :failed_permanently, :failed_transiently counts
       def process_batch
         ActiveRecord::Base.transaction do
           events = Event.fetch_batch_for_update
@@ -38,20 +43,24 @@ module Journaled
 
           result = batch_sender.send_batch(events)
 
-          # Delete successful events
           Event.where(id: result[:succeeded].map(&:id)).delete_all if result[:succeeded].any?
 
-          # Mark failed events
-          mark_events_as_failed(result[:failed]) if result[:failed].any?
+          permanent_failures = result[:failed].select(&:permanent?)
+          transient_failures = result[:failed].select(&:transient?)
+
+          mark_events_as_failed(permanent_failures) if permanent_failures.any?
 
           Rails.logger.info(
             "[journaled] Batch complete: #{result[:succeeded].count} succeeded, " \
-            "#{result[:failed].count} marked as failed (batch size: #{events.count})",
+            "#{permanent_failures.count} permanently failed, " \
+            "#{transient_failures.count} transiently failed (will retry) " \
+            "(batch size: #{events.count})",
           )
 
           {
             succeeded: result[:succeeded].count,
-            failed_permanently: result[:failed].count,
+            failed_permanently: permanent_failures.count,
+            failed_transiently: transient_failures.count,
           }
         end
       end

data/lib/journaled/outbox/metric_emitter.rb

@@ -2,82 +2,92 @@
 
 module Journaled
   module Outbox
-    # Handles metric emission for the Worker
+    # Handles metric emission for the Worker and Kinesis senders
     #
-    # This class is responsible for collecting and emitting metrics about the outbox queue.
+    # This class provides utility methods for collecting and emitting metrics.
     class MetricEmitter
-      def initialize(worker_id:)
-        @worker_id = worker_id
-      end
+      class << self
+        # Emit batch processing metrics
+        #
+        # @param stats [Hash] Processing statistics with :succeeded, :failed_permanently, :failed_transiently
+        # @param worker_id [String] ID of the worker processing the batch
+        def emit_batch_metrics(stats, worker_id:)
+          total_events = stats[:succeeded] + stats[:failed_permanently] + stats[:failed_transiently]
 
-      # Emit batch processing metrics
-      #
-      # @param stats [Hash] Processing statistics with :succeeded, :failed_permanently
-      def emit_batch_metrics(stats)
-        total_events = stats[:succeeded] + stats[:failed_permanently]
+          emit_metric('journaled.outbox_event.processed', value: total_events, worker_id:)
+          emit_metric('journaled.outbox_event.sent', value: stats[:succeeded], worker_id:)
+          emit_metric('journaled.outbox_event.failed', value: stats[:failed_permanently], worker_id:)
+          emit_metric('journaled.outbox_event.errored', value: stats[:failed_transiently], worker_id:)
+        end
 
-        emit_metric('journaled.worker.batch_process', value: total_events)
-        emit_metric('journaled.worker.batch_sent', value: stats[:succeeded])
-        emit_metric('journaled.worker.batch_failed', value: stats[:failed_permanently])
-      end
+        # Collect and emit queue metrics
+        #
+        # This calculates various queue statistics and emits individual metrics for each.
+        # @param worker_id [String] ID of the worker collecting metrics
+        def emit_queue_metrics(worker_id:)
+          metrics = calculate_queue_metrics
 
-      # Collect and emit queue metrics
-      #
-      # This calculates various queue statistics and emits individual metrics for each.
-      def emit_queue_metrics
-        metrics = calculate_queue_metrics
+          emit_metric('journaled.worker.queue_total_count', value: metrics[:total_count], worker_id:)
+          emit_metric('journaled.worker.queue_workable_count', value: metrics[:workable_count], worker_id:)
+          emit_metric('journaled.worker.queue_failed_count', value: metrics[:failed_count], worker_id:)
+          emit_metric('journaled.worker.queue_oldest_age_seconds', value: metrics[:oldest_age_seconds], worker_id:)
 
-        emit_metric('journaled.worker.queue_total_count', value: metrics[:total_count])
-        emit_metric('journaled.worker.queue_workable_count', value: metrics[:workable_count])
-        emit_metric('journaled.worker.queue_erroring_count', value: metrics[:erroring_count])
-        emit_metric('journaled.worker.queue_oldest_age_seconds', value: metrics[:oldest_age_seconds])
+          Rails.logger.info(
+            "Queue metrics: total=#{metrics[:total_count]}, " \
+            "workable=#{metrics[:workable_count]}, " \
+            "failed=#{metrics[:failed_count]}, " \
+            "oldest_age=#{metrics[:oldest_age_seconds].round(2)}s",
+          )
+        end
 
-        Rails.logger.info(
-          "Queue metrics: total=#{metrics[:total_count]}, " \
-          "workable=#{metrics[:workable_count]}, " \
-          "erroring=#{metrics[:erroring_count]}, " \
-          "oldest_age=#{metrics[:oldest_age_seconds].round(2)}s",
-        )
-      end
+        # Emit a metric notification for a Kinesis send failure
+        #
+        # @param event [Journaled::Outbox::Event] The failed event
+        # @param error_code [String] The error code (e.g., 'ProvisionedThroughputExceededException')
+        def emit_kinesis_failure(event:, error_code:)
+          emit_metric(
+            'journaled.kinesis.send_failure',
+            partition_key: event.partition_key,
+            error_code:,
+            stream_name: event.stream_name,
+            event_type: event.event_type,
+          )
+        end
 
-      private
+        private
 
-      attr_reader :worker_id
-
-      # Emit a single metric notification
-      #
-      # @param event_name [String] The name of the metric event
-      # @param payload [Hash] Additional payload data (event_count, value, etc.)
-      def emit_metric(event_name, payload)
-        ActiveSupport::Notifications.instrument(
-          event_name,
-          payload.merge(worker_id:),
-        )
-      end
+        # Emit a single metric notification
+        #
+        # @param event_name [String] The name of the metric event
+        # @param payload [Hash] Additional payload data (event_count, value, etc.)
+        def emit_metric(event_name, payload)
+          ActiveSupport::Notifications.instrument(event_name, payload)
+        end
 
-      # Calculate queue metrics
-      #
-      # @return [Hash] Metrics including counts and oldest event timestamp
-      def calculate_queue_metrics
-        # Use a single query with COUNT(*) FILTER to calculate all counts in one table scan
-        result = Event.connection.select_one(
-          Event.select(
-            'COUNT(*) AS total_count',
-            'COUNT(*) FILTER (WHERE failed_at IS NULL) AS workable_count',
-            'COUNT(*) FILTER (WHERE failure_reason IS NOT NULL AND failed_at IS NULL) AS erroring_count',
-            'MIN(created_at) FILTER (WHERE failed_at IS NULL) AS oldest_non_failed_timestamp',
-          ).to_sql,
-        )
+        # Calculate queue metrics
+        #
+        # @return [Hash] Metrics including counts and oldest event timestamp
+        def calculate_queue_metrics
+          # Use a single query with COUNT(*) FILTER to calculate all counts in one table scan
+          result = Event.connection.select_one(
+            Event.select(
+              'COUNT(*) AS total_count',
+              'COUNT(*) FILTER (WHERE failed_at IS NULL) AS workable_count',
+              'COUNT(*) FILTER (WHERE failure_reason IS NOT NULL AND failed_at IS NULL) AS failed_count',
+              'MIN(created_at) FILTER (WHERE failed_at IS NULL) AS oldest_non_failed_timestamp',
+            ).to_sql,
+          )
 
-        oldest_timestamp = result['oldest_non_failed_timestamp']
-        oldest_age_seconds = oldest_timestamp ? Time.current - oldest_timestamp : 0
+          oldest_timestamp = result['oldest_non_failed_timestamp']
+          oldest_age_seconds = oldest_timestamp ? Time.current - oldest_timestamp : 0
 
-        {
-          total_count: result['total_count'],
-          workable_count: result['workable_count'],
-          erroring_count: result['erroring_count'],
-          oldest_age_seconds:,
-        }
+          {
+            total_count: result['total_count'],
+            workable_count: result['workable_count'],
+            failed_count: result['failed_count'],
+            oldest_age_seconds:,
+          }
+        end
       end
     end
   end

data/lib/journaled/outbox/worker.rb

@@ -18,7 +18,6 @@ module Journaled
         @worker_id = "#{Socket.gethostname}-#{Process.pid}"
         self.running = false
         @processor = BatchProcessor.new
-        @metric_emitter = MetricEmitter.new(worker_id: @worker_id)
         self.shutdown_requested = false
         @last_metrics_emission = Time.current
       end
@@ -50,7 +49,7 @@ module Journaled
 
       private
 
-      attr_reader :worker_id, :processor, :metric_emitter
+      attr_reader :worker_id, :processor
       attr_accessor :shutdown_requested, :running, :last_metrics_emission
 
       def run_loop
@@ -60,9 +59,8 @@ module Journaled
             break
           end
 
-          events_processed = 0
           begin
-            events_processed = process_batch
+            process_batch
             emit_metrics_if_needed
           rescue StandardError => e
             Rails.logger.error("Worker error: #{e.class} - #{e.message}")
@@ -71,21 +69,14 @@ module Journaled
 
           break if shutdown_requested
 
-          # Only sleep if no events were processed to prevent excessive polling on empty table
-          sleep(Journaled.worker_poll_interval) if events_processed.zero?
+          sleep(Journaled.worker_poll_interval)
         end
       end
 
       def process_batch
         stats = processor.process_batch
 
-        instrument_batch_results(stats)
-
-        stats[:succeeded] + stats[:failed_permanently]
-      end
-
-      def instrument_batch_results(stats)
-        metric_emitter.emit_batch_metrics(stats)
+        MetricEmitter.emit_batch_metrics(stats, worker_id:)
       end
 
       def check_prerequisites!
@@ -128,7 +119,7 @@ module Journaled
 
       # Collect and emit queue metrics
       def collect_and_emit_metrics
-        metric_emitter.emit_queue_metrics
+        MetricEmitter.emit_queue_metrics(worker_id:)
       end
     end
   end

data/lib/journaled/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Journaled
-  VERSION = "6.2.1"
+  VERSION = "6.2.3"
 end

data/lib/journaled.rb

@@ -12,7 +12,9 @@ require 'journaled/delivery_adapter'
 require 'journaled/delivery_adapters/active_job_adapter'
 require 'journaled/outbox/adapter'
 require 'journaled/kinesis_client_factory'
+require 'journaled/kinesis_failed_event'
 require 'journaled/kinesis_batch_sender'
+require 'journaled/kinesis_sequential_sender'
 require 'journaled/outbox/batch_processor'
 require 'journaled/outbox/metric_emitter'
 require 'journaled/outbox/worker'
@@ -31,8 +33,9 @@ module Journaled
   mattr_writer(:transactional_batching_enabled) { true }
 
   # Worker configuration (for Outbox-style event processing)
-  mattr_accessor(:worker_batch_size) { 1000 }
-  mattr_accessor(:worker_poll_interval) { 1 } # seconds
+  mattr_accessor(:worker_batch_size) { 500 }
+  mattr_accessor(:worker_poll_interval) { 0.5 } # seconds
+  mattr_accessor(:outbox_processing_mode) { :batch } # :batch or :guaranteed_order
 
   def self.transactional_batching_enabled?
     Thread.current[:journaled_transactional_batching_enabled] || @@transactional_batching_enabled

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: journaled
 version: !ruby/object:Gem::Version
-  version: 6.2.1
+  version: 6.2.3
 platform: ruby
 authors:
 - Jake Lipson
@@ -274,6 +274,8 @@ files:
 - lib/journaled/errors.rb
 - lib/journaled/kinesis_batch_sender.rb
 - lib/journaled/kinesis_client_factory.rb
+- lib/journaled/kinesis_failed_event.rb
+- lib/journaled/kinesis_sequential_sender.rb
 - lib/journaled/outbox/adapter.rb
 - lib/journaled/outbox/batch_processor.rb
 - lib/journaled/outbox/metric_emitter.rb