dionysus-rb 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d190bbc3fd4b2ca0aa7c7d1be7c0ddaa6a3b6bdcbd6556b90e4752c2e875bfc6
-  data.tar.gz: 33379f5ccc97bceee7caad1d9b3383e84c3073e48b15bda1d0f592e765c39352
+  metadata.gz: 715bda096eaf87ea615134206607a11d22ff3d673b5cbe9625c584599fb5696b
+  data.tar.gz: d4c5741f509d02f0c5a28a4332f34b08b28f46ca1c7d026d59f59bff2e37638f
 SHA512:
-  metadata.gz: cafb4cb76ad00f5dbbc45f9c3a578284ef554aaf680310286961d1f8251620bc67c10e190f301ac06fca1c8e55be9fab2e75977cc85ecf80f21503504558da20
-  data.tar.gz: 78e8c07d652f3dcc46cb3a527834cb72b2290b119266e4362cde3ddbf1d5679c147cb03cce34acb6682451c6453fb58532faab8a7bf5be43a6db6be791e7e1e2
+  metadata.gz: c101ce0cfa0f3ddb58339bdf28b3b479838fb3b66107d8223d4a795888e5048cc8f044db232fe7ae03899cf558d1b8100a8119119d8a8d3faac3996082671618
+  data.tar.gz: be3c63dddbc4b1c3ae58134b050370c62a0940d3227ca3e1820cce72a1bba36cb932de2906d42a7fe723f811ac30dbc4431f668c5832787fd0a27fb521fc5594

data/.rubocop.yml

@@ -170,6 +170,7 @@ Naming/AccessorMethodName:
 RSpec/DescribeClass:
   Exclude:
     - 'spec/integration_spec.rb'
+    - 'spec/dedup_serialization_time_integration_spec.rb'
 
 Style/MultilineBlockChain:
   Exclude:

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## [Unreleased]
 
+## [1.2.0]
+
+### Fixed
+
+- Rank duplicate messages in a consumer batch by when their payload was serialized, not by Kafka offset. The previous release ranked on the highest offset, on the premise that offsets strictly increase in publish order. They do - but publish order is not serialization order: `Publisher#publish` does `find_by` -> `generate_message` -> `responder.call`, so a message produced later can carry an earlier snapshot. Measured in production on one record, eight `updated` events inside 700 ms: the offset the batch ended on was published at `.905` carrying a payload read at `.398194` with `paid_amount 0.0`, while an earlier offset published at `.837` carried a payload read at `.671830` with the real value of `1447.0`. Ranking on offset kept the stale payload and the consumer's value stuck at `0.00` with nothing to self-heal it.
+- Neither field already in the message is a freshness signal. `updated_at` is the record's own timestamp and does not move when only an associated record changes, so two payloads with different computed values can share one `updated_at` - that is the tie the previous release was written for. The offset is publish order, per above. The two known incidents are mirror images: in one the correct message has the higher offset and the older `updated_at`, in the other the lower offset and the newer `updated_at`, so no ordering over the existing fields fixes both.
+- The producer therefore stamps `serialized_at` where the payload is actually built, and the consumer ranks on `[serialized_at, offset]`. Messages without the field fall back to `Time.at(0)` and are ordered among themselves by offset, which is exactly the previous behaviour; a stamped message wins over an unstamped one, which is correct because a stamp can only come from a newer producer.
+- The stamp is gated on `config.include_serialized_at_in_payload`, default `false`, so published envelopes are unchanged until it is switched on. Roll consumers out first - with the fallback that deploy is a no-op - then enable it on the producer.
+
 ## [1.1.0]
 
 ### Fixed

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dionysus-rb (1.1.0)
+    dionysus-rb (1.2.0)
       activerecord (>= 5)
       activesupport (>= 3.2)
       concurrent-ruby

data/lib/dionysus/consumer/params_batch_transformations/remove_duplicates_strategy.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
+require "time"
+
 class Dionysus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy
+  SERIALIZED_AT_FORMAT = %r{\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z\z}
+
   def call(params_batch)
     return params_batch if duplicates_removal_not_applicable?(params_batch)
 
@@ -12,17 +16,37 @@ class Dionysus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy
   # the idea is following:
   # 1. group messages by event and id - for a given model we can expect unique messages for _created and _deleted events
   #   but we can have multiple _updated events, so this is where we are interested in removing duplicates
-  # 2. from each group keep the message published last (highest offset) - payloads are serialized at
-  #   publish time, so that is the freshest state; "updated_at" does not track it
+  # 2. from each group keep the message whose payload was serialized last, with the offset as a tie-break -
+  #   the offset alone is publish order, and a message published later can carry an earlier snapshot
   # 3. flatten the array of arrays as all groups will have a single item
   def transform_messages_array(params_batch)
     params_batch
       .to_a
       .group_by { |batch| grouping_key_by_event_and_id(batch) }
-      .map { |_, group| group.max_by(&:offset) }
+      .map { |_, group| freshest(group) }
       .flatten
   end
 
+  # a group is ranked by its stamps only when every message in it carries one - during the
+  # producer rollout a batch can hold both, and a stamp says nothing about an unstamped neighbour
+  def freshest(group)
+    stamps = group.map { |message| serialized_at_for(message) }
+
+    return group.max_by(&:offset) if stamps.any?(&:nil?)
+
+    group.zip(stamps).max_by { |message, serialized_at| [serialized_at, message.offset] }.first
+  end
+
+  def serialized_at_for(message)
+    raw = message.payload.fetch("message").first["serialized_at"]
+
+    return unless raw.is_a?(String) && raw.match?(SERIALIZED_AT_FORMAT)
+
+    Time.parse(raw)
+  rescue
+    nil
+  end
+
   def grouping_key_by_event_and_id(batch)
     [
       batch.payload.fetch("message").first.fetch("event"),

data/lib/dionysus/producer/config.rb

@@ -8,7 +8,8 @@ class Dionysus::Producer::Config
     :lock_client, :lock_expiry_time, :error_handler, :outbox_publishing_batch_size,
     :transactional_outbox_enabled, :sidekiq_queue, :publisher_service_name,
     :genesis_consistency_safety_delay, :hermes_event_producer, :publish_after_commit, :outbox_worker_publishing_delay,
-    :high_priority_sidekiq_queue, :observers_inline_maximum_size, :remove_consecutive_duplicates_before_publishing
+    :high_priority_sidekiq_queue, :observers_inline_maximum_size, :remove_consecutive_duplicates_before_publishing,
+    :include_serialized_at_in_payload
 
   def self.default_sidekiq_queue
     :dionysus
@@ -96,6 +97,14 @@ class Dionysus::Producer::Config
     @observers_inline_maximum_size || 1000
   end
 
+  # Off by default so consumers, which fall back to the offset when the field is absent, can be
+  # rolled out first.
+  def include_serialized_at_in_payload
+    return @include_serialized_at_in_payload if defined?(@include_serialized_at_in_payload)
+
+    false
+  end
+
   def remove_consecutive_duplicates_before_publishing
     return @remove_consecutive_duplicates_before_publishing if defined?(@remove_consecutive_duplicates_before_publishing)
 

data/lib/dionysus/producer/karafka_responder_generator.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "active_support/core_ext/string"
+require "time"
 
 class Dionysus::Producer::KarafkaResponderGenerator
   TOMBSTONE = nil
@@ -45,13 +46,18 @@ class Dionysus::Producer::KarafkaResponderGenerator
 
               record = records.sample
 
+              # the offset alone is publish order, and a message published later can carry an
+              # earlier snapshot, so consumers need to know when this payload was actually read
+              serialized_at = config.include_serialized_at_in_payload ? Time.now.utc : nil
               payload = serialize_to_payload(records, topic, batch_options)
 
-              {
+              event_payload = {
                 event: event,
                 model_name: record.model_name.name,
                 data: payload
               }
+              event_payload[:serialized_at] = serialized_at.iso8601(6) if serialized_at
+              event_payload
             end
             unless genesis_only?(options)
               respond_to topic_name, { message: message }, **final_options

data/lib/dionysus/version.rb

@@ -3,5 +3,5 @@
 module Dionysus
   module Version
   end
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dionysus-rb
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-18 00:00:00.000000000 Z
+date: 2026-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord