dionysus-rb 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c0362ffd0074956ac7ace1f3c4a392eaa7b5b09a79ecf113a14f9da8f9f5b22
-  data.tar.gz: 44cdab85d128017c09dc0535c1225d9d68515addc5c2d1d4b28b50dbb2fc0150
+  metadata.gz: 715bda096eaf87ea615134206607a11d22ff3d673b5cbe9625c584599fb5696b
+  data.tar.gz: d4c5741f509d02f0c5a28a4332f34b08b28f46ca1c7d026d59f59bff2e37638f
 SHA512:
-  metadata.gz: 63905f2eb238cdca13e750e728498449d6238be26f30bbe68c4709c61218c989d33dd8c7da553c4c4ec6e36dc6b05cc84a8cc44067048752b88018338d1809ee
-  data.tar.gz: cabf8edf52bd2ba5ffaf26608110a7f110ba4a7c15ff2ead082c74cd780c5018bd92b0b44a15e847edcf65c39a898bdd537f2c9c85c3c191c1a1803c1a657e96
+  metadata.gz: c101ce0cfa0f3ddb58339bdf28b3b479838fb3b66107d8223d4a795888e5048cc8f044db232fe7ae03899cf558d1b8100a8119119d8a8d3faac3996082671618
+  data.tar.gz: be3c63dddbc4b1c3ae58134b050370c62a0940d3227ca3e1820cce72a1bba36cb932de2906d42a7fe723f811ac30dbc4431f668c5832787fd0a27fb521fc5594

data/.rubocop.yml

@@ -170,6 +170,7 @@ Naming/AccessorMethodName:
 RSpec/DescribeClass:
   Exclude:
     - 'spec/integration_spec.rb'
+    - 'spec/dedup_serialization_time_integration_spec.rb'
 
 Style/MultilineBlockChain:
   Exclude:

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 ## [Unreleased]
 
+## [1.2.0]
+
+### Fixed
+
+- Rank duplicate messages in a consumer batch by when their payload was serialized, not by Kafka offset. The previous release ranked on the highest offset, on the premise that offsets strictly increase in publish order. They do - but publish order is not serialization order: `Publisher#publish` does `find_by` -> `generate_message` -> `responder.call`, so a message produced later can carry an earlier snapshot. Measured in production on one record, eight `updated` events inside 700 ms: the offset the batch ended on was published at `.905` carrying a payload read at `.398194` with `paid_amount 0.0`, while an earlier offset published at `.837` carried a payload read at `.671830` with the real value of `1447.0`. Ranking on offset kept the stale payload and the consumer's value stuck at `0.00` with nothing to self-heal it.
+- Neither field already in the message is a freshness signal. `updated_at` is the record's own timestamp and does not move when only an associated record changes, so two payloads with different computed values can share one `updated_at` - that is the tie the previous release was written for. The offset is publish order, per above. The two known incidents are mirror images: in one the correct message has the higher offset and the older `updated_at`, in the other the lower offset and the newer `updated_at`, so no ordering over the existing fields fixes both.
+- The producer therefore stamps `serialized_at` where the payload is actually built, and the consumer ranks on `[serialized_at, offset]`. Messages without the field fall back to `Time.at(0)` and are ordered among themselves by offset, which is exactly the previous behaviour; a stamped message wins over an unstamped one, which is correct because a stamp can only come from a newer producer.
+- The stamp is gated on `config.include_serialized_at_in_payload`, default `false`, so published envelopes are unchanged until it is switched on. Roll consumers out first - with the fallback that deploy is a no-op - then enable it on the producer.
+
+## [1.1.0]
+
+### Fixed
+
+- Keep the last published message when removing duplicates from a consumer batch. `RemoveDuplicatesStrategy` kept the message with the highest payload `updated_at` per event and id, but payloads are serialized at publish time, so freshness follows publish order rather than `updated_at`: a payload carrying values computed on read from associated records changes without the record's own `updated_at` changing, and can then arrive with an older `updated_at` than a staler message. Those messages were silently discarded, losing the change in every consumer with no error and no trace. The message with the highest Kafka offset is kept instead - a Karafka batch comes from a single topic-partition, so offsets strictly increase in publish order.
+
 ## [1.0.0]
 
 Brings `dionysus-rb` back in line with the private gem it was extracted from, which had moved on

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dionysus-rb (1.0.0)
+    dionysus-rb (1.2.0)
       activerecord (>= 5)
       activesupport (>= 3.2)
       concurrent-ruby

data/lib/dionysus/consumer/params_batch_transformations/remove_duplicates_strategy.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
+require "time"
+
 class Dionysus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy
+  SERIALIZED_AT_FORMAT = %r{\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z\z}
+
   def call(params_batch)
     return params_batch if duplicates_removal_not_applicable?(params_batch)
 
@@ -10,21 +14,39 @@ class Dionysus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy
   private
 
   # the idea is following:
-  # 1. group messages by event and id - for a given model we can expect unique messages for _created and _deleted ecents
+  # 1. group messages by event and id - for a given model we can expect unique messages for _created and _deleted events
   #   but we can have multiple _updated events, so this is where we are interested in removing duplicates
-  # 2. sort each group by updated_at, reverse (because the sort is ascending) and take the first one -
-  #   this way we will have the most recent update
+  # 2. from each group keep the message whose payload was serialized last, with the offset as a tie-break -
+  #   the offset alone is publish order, and a message published later can carry an earlier snapshot
   # 3. flatten the array of arrays as all groups will have a single item
-  # It is safer to apply sorting just for the _updated events to a given model as otherwise we could change the order of
-  #   the messages for a different type which might be not desirable
   def transform_messages_array(params_batch)
     params_batch
       .to_a
       .group_by { |batch| grouping_key_by_event_and_id(batch) }
-      .map { |_, group| group.max_by { |batch| updated_at_from_batch(batch) } }
+      .map { |_, group| freshest(group) }
       .flatten
   end
 
+  # a group is ranked by its stamps only when every message in it carries one - during the
+  # producer rollout a batch can hold both, and a stamp says nothing about an unstamped neighbour
+  def freshest(group)
+    stamps = group.map { |message| serialized_at_for(message) }
+
+    return group.max_by(&:offset) if stamps.any?(&:nil?)
+
+    group.zip(stamps).max_by { |message, serialized_at| [serialized_at, message.offset] }.first
+  end
+
+  def serialized_at_for(message)
+    raw = message.payload.fetch("message").first["serialized_at"]
+
+    return unless raw.is_a?(String) && raw.match?(SERIALIZED_AT_FORMAT)
+
+    Time.parse(raw)
+  rescue
+    nil
+  end
+
   def grouping_key_by_event_and_id(batch)
     [
       batch.payload.fetch("message").first.fetch("event"),
@@ -32,14 +54,6 @@ class Dionysus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy
     ].join
   end
 
-  def updated_at_from_batch(batch)
-    timestamp = batch.payload.fetch("message", []).first.to_h.fetch("data", []).first.to_h.fetch("updated_at", nil)
-
-    return timestamp.to_datetime if timestamp.respond_to?(:to_datetime)
-
-    Time.current
-  end
-
   def duplicates_removal_not_applicable?(params_batch)
     any_message_containing_more_than_one_event?(params_batch) || any_event_containing_more_than_one_item(params_batch)
   end

data/lib/dionysus/producer/config.rb

@@ -8,7 +8,8 @@ class Dionysus::Producer::Config
     :lock_client, :lock_expiry_time, :error_handler, :outbox_publishing_batch_size,
     :transactional_outbox_enabled, :sidekiq_queue, :publisher_service_name,
     :genesis_consistency_safety_delay, :hermes_event_producer, :publish_after_commit, :outbox_worker_publishing_delay,
-    :high_priority_sidekiq_queue, :observers_inline_maximum_size, :remove_consecutive_duplicates_before_publishing
+    :high_priority_sidekiq_queue, :observers_inline_maximum_size, :remove_consecutive_duplicates_before_publishing,
+    :include_serialized_at_in_payload
 
   def self.default_sidekiq_queue
     :dionysus
@@ -96,6 +97,14 @@ class Dionysus::Producer::Config
     @observers_inline_maximum_size || 1000
   end
 
+  # Off by default so consumers, which fall back to the offset when the field is absent, can be
+  # rolled out first.
+  def include_serialized_at_in_payload
+    return @include_serialized_at_in_payload if defined?(@include_serialized_at_in_payload)
+
+    false
+  end
+
   def remove_consecutive_duplicates_before_publishing
     return @remove_consecutive_duplicates_before_publishing if defined?(@remove_consecutive_duplicates_before_publishing)
 

data/lib/dionysus/producer/karafka_responder_generator.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "active_support/core_ext/string"
+require "time"
 
 class Dionysus::Producer::KarafkaResponderGenerator
   TOMBSTONE = nil
@@ -45,13 +46,18 @@ class Dionysus::Producer::KarafkaResponderGenerator
 
               record = records.sample
 
+              # the offset alone is publish order, and a message published later can carry an
+              # earlier snapshot, so consumers need to know when this payload was actually read
+              serialized_at = config.include_serialized_at_in_payload ? Time.now.utc : nil
               payload = serialize_to_payload(records, topic, batch_options)
 
-              {
+              event_payload = {
                 event: event,
                 model_name: record.model_name.name,
                 data: payload
               }
+              event_payload[:serialized_at] = serialized_at.iso8601(6) if serialized_at
+              event_payload
             end
             unless genesis_only?(options)
               respond_to topic_name, { message: message }, **final_options

data/lib/dionysus/version.rb

@@ -3,5 +3,5 @@
 module Dionysus
   module Version
   end
-  VERSION = "1.0.0"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,13 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dionysus-rb
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Karol Galanciak
+autorequire:
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2026-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -563,6 +564,7 @@ metadata:
   source_code_uri: https://github.com/BookingSync/dionysus-rb
   changelog_uri: https://github.com/BookingSync/dionysus-rb/blob/master/CHANGELOG.md
   rubygems_mfa_required: 'true'
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -577,7 +579,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.18
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: A framework on top of Karafka for Change Data Capture on the model level.
 test_files: []