logister-ruby 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d1ca80fe29bc17af3e645e1a07ac8f4d68bd5c7f60b01d238ef9e875e8d60c3
-  data.tar.gz: 73418842ba1d3a221374a87313e2147b70473a85fb6509132a1213832a1c79c3
+  metadata.gz: ff89324842289ec0925e84131ee9442d4dfed5589b86deb1bb56498e55eb5977
+  data.tar.gz: f262d7ab245a8c5703fb28873e65d4ee11194be7146833f656fd74d85434f2ff
 SHA512:
-  metadata.gz: 4f99665d9daa0761f7a761470c07174b8f181791caa1639a40324650b615f84932a440e9f037133221047389b64e1ffa1747c7c6dc3e661c2626e4f9978aac38
-  data.tar.gz: 767b7c876a8d8a05ed0dbbf106de09ccd474d2b1796ff5ec82e6194f9da27e1aa4abdd765f17fd3c20c71a0478ba10460250829f5e056415c98da495a8049dc1
+  metadata.gz: 28a1b3621688f111349b37c9996d530855d07081c17ce4525e33426b2ff30f1c714f8871555925dd7c11ec78ff09ad0566295552c54d05d141c28b944fe2f383
+  data.tar.gz: c262d8b152f55680dc7ad02d19733089b59415f4a853cf33d5e8ffcade2851a69d4ada2963a7e05e4af8ca758e08340288213ea5e90da68d68c1b6baaf768108

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## v0.4.0 - 2026-08-08
+
+- Added stable UUID assignment before synchronous or asynchronous delivery so retries preserve one logical event identity.
+- Added configurable gzip/NDJSON batching for the Logister batch-ingest endpoint, including deterministic batch IDs, bounded batch size and interval, and automatic fallback to stable single-event delivery for older servers.
+- Added recursive `413` batch splitting while continuing every split or fallback attempt, preventing one failed subset from short-circuiting the remainder.
+- Added transient-only retries with connect, read, and write timeouts, numeric or HTTP-date `Retry-After` support, capped exponential backoff, and bounded jitter.
+- Added batch and retry settings to the Rails generator and Railtie configuration bridge, and made `Logister.flush` wait for in-flight batch delivery rather than only an empty queue.
+
 ## v0.3.1 - 2026-07-26
 
 - Added `Logister.suppress_reporting` and `Logister.reporting_suppressed?` for recursion-safe telemetry processing, including early exits in automatic SQL and request subscribers.

data/README.md

@@ -146,11 +146,26 @@ Keep `LOGISTER_API_KEY` in your deployment secret store. Project API keys are wr
 Logister.configure do |config|
   config.async = true
   config.queue_size = 1000
+  config.batch_size = 50
+  config.batch_interval = 0.05
+  config.batch_compression = true
   config.max_retries = 3
   config.retry_base_interval = 0.5
+  config.max_retry_delay = 30.0
+  config.retry_jitter = 0.2
 end
 ```
 
+Asynchronous delivery assigns a UUID before enqueueing, combines queued events into
+gzip/NDJSON batches, and retries the same identifiers. This makes a whole-batch retry
+safe against a Logister server that supports the batch endpoint. Older servers are
+detected automatically and receive the same stable events through the single-event
+endpoint. Call `Logister.flush` before a short-lived process exits.
+
+Every HTTP attempt applies `timeout_seconds` to connect, read, and write operations.
+Retryable responses honor `Retry-After` when present, cap any individual wait at
+`max_retry_delay`, and add bounded positive jitter controlled by `retry_jitter`.
+
 ## Filtering and redaction
 
 ```ruby

data/lib/generators/logister/templates/logister.rb

@@ -10,8 +10,13 @@ Logister.configure do |config|
 
   config.async = true
   config.queue_size = 1000
+  config.batch_size = 50
+  config.batch_interval = 0.05
+  config.batch_compression = true
   config.max_retries = 3
   config.retry_base_interval = 0.5
+  config.max_retry_delay = 30.0
+  config.retry_jitter = 0.2
 
   config.ignore_environments = []
   config.ignore_exceptions = []

data/lib/logister/client.rb

@@ -2,11 +2,29 @@
 
 require 'json'
 require 'net/http'
+require 'digest'
+require 'securerandom'
+require 'time'
 require 'uri'
+require 'zlib'
 
 module Logister
   class Client
     CONTENT_TYPE = 'application/json'
+    BATCH_CONTENT_TYPE = 'application/x-ndjson'
+    UNSUPPORTED_BATCH_STATUSES = %w[404 405 415 501].freeze
+
+    class UnsupportedBatchEndpoint < StandardError; end
+
+    class RequestError < StandardError
+      attr_reader :status, :retry_after
+
+      def initialize(status, retry_after: nil)
+        @status = status.to_i
+        @retry_after = retry_after
+        super("HTTP #{status}")
+      end
+    end
 
     def initialize(configuration)
       @configuration = configuration
@@ -14,12 +32,17 @@ module Logister
       @queue         = SizedQueue.new(@configuration.queue_size)
       @worker        = nil
       @running       = false
+      @pending_mutex = Mutex.new
+      @pending_condition = ConditionVariable.new
+      @pending_count = 0
 
       # Cache values that are static for the lifetime of this client so we
       # don't allocate on every send_request call.
       @uri            = URI.parse(@configuration.endpoint).freeze
+      @batch_uri      = URI.parse(@configuration.batch_endpoint).freeze
       @deployment_uri = URI.parse(@configuration.deployment_endpoint).freeze
       @use_ssl        = @uri.scheme == 'https'
+      @batch_use_ssl  = @batch_uri.scheme == 'https'
       @deployment_use_ssl = @deployment_uri.scheme == 'https'
       @auth_header    = "Bearer #{@configuration.api_key}".freeze
     end
@@ -27,6 +50,8 @@ module Logister
     def publish(payload)
       return false unless ready?
 
+      payload = with_stable_uuid(payload)
+
       return publish_sync(payload) unless @configuration.async
 
       ensure_worker_started
@@ -43,10 +68,13 @@ module Logister
       return true unless @configuration.async
 
       deadline = monotonic_now + timeout
-      until @queue.empty?
-        return false if monotonic_now > deadline
+      @pending_mutex.synchronize do
+        while @pending_count.positive?
+          remaining = deadline - monotonic_now
+          return false unless remaining.positive?
 
-        sleep(0.01)
+          @pending_condition.wait(@pending_mutex, [remaining, 0.05].min)
+        end
       end
 
       true
@@ -69,9 +97,11 @@ module Logister
     private
 
     def enqueue(payload)
+      increment_pending
       @queue.push(payload, true)
       true
     rescue ThreadError
+      complete_pending(1)
       @configuration.logger.warn('logister queue full; dropping event')
       false
     end
@@ -90,11 +120,31 @@ module Logister
     end
 
     def run_worker
+      stop_after_batch = false
       loop do
         payload = @queue.pop
         break if payload.nil?
 
-        publish_sync(payload)
+        batch = [payload]
+        deadline = monotonic_now + batch_interval
+
+        while batch.length < batch_size && monotonic_now < deadline
+          begin
+            queued = @queue.pop(true)
+            if queued.nil?
+              stop_after_batch = true
+              break
+            end
+            batch << queued
+          rescue ThreadError
+            remaining = deadline - monotonic_now
+            sleep([remaining, 0.005].min) if remaining.positive?
+          end
+        end
+
+        publish_batch_sync(batch)
+        complete_pending(batch.length)
+        break if stop_after_batch
       end
     rescue StandardError => e
       @configuration.logger.warn("logister worker crashed: #{e.class} #{e.message}")
@@ -110,8 +160,8 @@ module Logister
         attempts += 1
         send_request(payload)
       rescue StandardError => e
-        if attempts <= @configuration.max_retries
-          sleep(@configuration.retry_base_interval * (2**(attempts - 1)))
+        if attempts <= @configuration.max_retries && retryable_error?(e)
+          sleep(retry_delay(e, attempts))
           retry
         end
 
@@ -120,14 +170,46 @@ module Logister
       end
     end
 
+    def publish_batch_sync(payloads)
+      attempts = 0
+      begin
+        attempts += 1
+        send_batch_request(payloads)
+      rescue UnsupportedBatchEndpoint
+        payloads.map { |payload| publish_sync(payload) }.all?
+      rescue RequestError => e
+        if e.status == 413 && payloads.length > 1
+          middle = (payloads.length / 2.0).ceil
+          first_half_delivered = publish_batch_sync(payloads.first(middle))
+          second_half_delivered = publish_batch_sync(payloads.drop(middle))
+          return first_half_delivered && second_half_delivered
+        end
+        if attempts <= @configuration.max_retries && retryable_error?(e)
+          sleep(retry_delay(e, attempts))
+          retry
+        end
+
+        @configuration.logger.warn("logister batch publish failed: #{e.class} #{e.message}")
+        false
+      rescue StandardError => e
+        if attempts <= @configuration.max_retries && retryable_error?(e)
+          sleep(retry_delay(e, attempts))
+          retry
+        end
+
+        @configuration.logger.warn("logister batch publish failed: #{e.class} #{e.message}")
+        false
+      end
+    end
+
     def publish_deployment_sync(payload)
       attempts = 0
       begin
         attempts += 1
         send_deployment_request(payload)
       rescue StandardError => e
-        if attempts <= @configuration.max_retries
-          sleep(@configuration.retry_base_interval * (2**(attempts - 1)))
+        if attempts <= @configuration.max_retries && retryable_error?(e)
+          sleep(retry_delay(e, attempts))
           retry
         end
 
@@ -147,12 +229,41 @@ module Logister
         @uri.port,
         use_ssl:      @use_ssl,
         open_timeout: @configuration.timeout_seconds,
-        read_timeout: @configuration.timeout_seconds
+        read_timeout: @configuration.timeout_seconds,
+        write_timeout: @configuration.timeout_seconds
+      ) { |http| http.request(request) }
+
+      return true if response.is_a?(Net::HTTPSuccess)
+
+      raise request_error(response)
+    end
+
+    def send_batch_request(payloads)
+      request = Net::HTTP::Post.new(@batch_uri)
+      request['Content-Type'] = BATCH_CONTENT_TYPE
+      request['Authorization'] = @auth_header
+      request['X-Logister-Batch-Id'] = batch_id(payloads)
+
+      body = payloads.map { |payload| { event: payload }.to_json }.join("\n") << "\n"
+      if @configuration.batch_compression
+        request['Content-Encoding'] = 'gzip'
+        body = Zlib.gzip(body)
+      end
+      request.body = body
+
+      response = Net::HTTP.start(
+        @batch_uri.host,
+        @batch_uri.port,
+        use_ssl: @batch_use_ssl,
+        open_timeout: @configuration.timeout_seconds,
+        read_timeout: @configuration.timeout_seconds,
+        write_timeout: @configuration.timeout_seconds
       ) { |http| http.request(request) }
 
       return true if response.is_a?(Net::HTTPSuccess)
+      raise UnsupportedBatchEndpoint, "HTTP #{response.code}" if UNSUPPORTED_BATCH_STATUSES.include?(response.code)
 
-      raise "HTTP #{response.code}"
+      raise request_error(response)
     end
 
     def send_deployment_request(payload)
@@ -166,18 +277,99 @@ module Logister
         @deployment_uri.port,
         use_ssl:      @deployment_use_ssl,
         open_timeout: @configuration.timeout_seconds,
-        read_timeout: @configuration.timeout_seconds
+        read_timeout: @configuration.timeout_seconds,
+        write_timeout: @configuration.timeout_seconds
       ) { |http| http.request(request) }
 
       return true if response.is_a?(Net::HTTPSuccess)
 
-      raise "HTTP #{response.code}"
+      raise request_error(response)
     end
 
     def monotonic_now
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
 
+    def with_stable_uuid(payload)
+      attributes = payload.to_h.dup
+      key = attributes.keys.any? { |candidate| candidate.is_a?(String) } ? 'uuid' : :uuid
+      uuid = [attributes[:uuid], attributes['uuid']].find { |value| !blank_identifier?(value) }
+      event_id = [attributes[:event_id], attributes['event_id']].find { |value| !blank_identifier?(value) }
+
+      attributes.delete(:uuid)
+      attributes.delete('uuid')
+      attributes.delete(:event_id) if blank_identifier?(attributes[:event_id])
+      attributes.delete('event_id') if blank_identifier?(attributes['event_id'])
+      attributes[key] = uuid || event_id || SecureRandom.uuid
+      attributes
+    end
+
+    def blank_identifier?(value)
+      value.nil? || value.to_s.strip.empty?
+    end
+
+    def batch_id(payloads)
+      identifiers = payloads.map do |payload|
+        payload[:uuid] || payload['uuid'] || payload[:event_id] || payload['event_id']
+      end
+      Digest::SHA256.hexdigest(identifiers.join("\n"))
+    end
+
+    def batch_size
+      [@configuration.batch_size.to_i, 1].max
+    end
+
+    def batch_interval
+      [@configuration.batch_interval.to_f, 0.0].max
+    end
+
+    def increment_pending
+      @pending_mutex.synchronize { @pending_count += 1 }
+    end
+
+    def complete_pending(count)
+      @pending_mutex.synchronize do
+        @pending_count = [@pending_count - count, 0].max
+        @pending_condition.broadcast if @pending_count.zero?
+      end
+    end
+
+    def retryable_error?(error)
+      return true unless error.is_a?(RequestError)
+
+      [408, 425, 429].include?(error.status) || error.status >= 500
+    end
+
+    def request_error(response)
+      RequestError.new(response.code, retry_after: retry_after_seconds(response['Retry-After']))
+    end
+
+    def retry_after_seconds(value)
+      header = value.to_s.strip
+      return nil if header.empty?
+
+      seconds = Float(header, exception: false)
+      return [seconds, 0.0].max if seconds&.finite?
+
+      [Time.httpdate(header) - Time.now, 0.0].max
+    rescue ArgumentError
+      nil
+    end
+
+    def retry_delay(error, attempt)
+      configured_cap = [@configuration.max_retry_delay.to_f, 0.0].max
+      base_delay = if error.is_a?(RequestError) && !error.retry_after.nil?
+                     error.retry_after.to_f
+                   else
+                     @configuration.retry_base_interval.to_f * (2**(attempt - 1))
+                   end
+      bounded_delay = [[base_delay, 0.0].max, configured_cap].min
+      jitter_ratio = @configuration.retry_jitter.to_f.clamp(0.0, 1.0)
+      jitter = bounded_delay * jitter_ratio * rand
+
+      [bounded_delay + jitter, configured_cap].min
+    end
+
     def ready?
       @configuration.enabled && !@configuration.api_key.to_s.empty?
     end

data/lib/logister/configuration.rb

@@ -6,11 +6,12 @@ module Logister
                   :repository, :commit_sha, :branch, :enabled, :timeout_seconds, :logger,
                   :ignore_exceptions, :ignore_environments, :ignore_paths, :before_notify,
                   :async, :queue_size, :max_retries, :retry_base_interval,
+                  :max_retry_delay, :retry_jitter, :batch_size, :batch_interval, :batch_compression,
                   :capture_db_metrics, :db_metric_min_duration_ms, :db_metric_sample_rate,
                   :feature_flags_resolver, :dependency_resolver, :anonymize_ip,
                   :max_breadcrumbs, :max_dependencies, :capture_request_spans,
                   :capture_sql_breadcrumbs, :sql_breadcrumb_min_duration_ms
-    attr_writer :deployment_endpoint
+    attr_writer :deployment_endpoint, :batch_endpoint
 
     def initialize
       @api_key = ENV['LOGISTER_API_KEY']
@@ -36,6 +37,11 @@ module Logister
       @queue_size = 1000
       @max_retries = 3
       @retry_base_interval = 0.5
+      @max_retry_delay = 30.0
+      @retry_jitter = 0.2
+      @batch_size = 50
+      @batch_interval = 0.05
+      @batch_compression = true
 
       @capture_db_metrics = false
       @db_metric_min_duration_ms = 0.0
@@ -55,6 +61,10 @@ module Logister
       @deployment_endpoint || endpoint.to_s.sub(%r{/ingest_events\z}, '/deployments')
     end
 
+    def batch_endpoint
+      @batch_endpoint || endpoint.to_s.sub(%r{/ingest_events\z}, '/ingest_events/batch')
+    end
+
     private
 
     def env_value(name)

data/lib/logister/railtie.rb

@@ -20,8 +20,13 @@ module Logister
         copy_setting(app, config, :before_notify)
         copy_setting(app, config, :async)
         copy_setting(app, config, :queue_size)
+        copy_setting(app, config, :batch_size)
+        copy_setting(app, config, :batch_interval)
+        copy_setting(app, config, :batch_compression)
         copy_setting(app, config, :max_retries)
         copy_setting(app, config, :retry_base_interval)
+        copy_setting(app, config, :max_retry_delay)
+        copy_setting(app, config, :retry_jitter)
         copy_setting(app, config, :capture_db_metrics)
         copy_setting(app, config, :db_metric_min_duration_ms)
         copy_setting(app, config, :db_metric_sample_rate)

data/lib/logister/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Logister
-  VERSION = '0.3.1'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logister-ruby
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Logister