dionysus-rb 1.6.0 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b3fb6e16f40d4396b115342f72098bbb2b3f56d8fb630898123e63acce866dc
-  data.tar.gz: ff8f56aa9b8a142131d92b53d83f7014b987439158c39a7518c096f9e05de980
+  metadata.gz: a103f61431f824a5d8769b0d2ed6d3ab6b9493690b3503726ed87d8790da7dd9
+  data.tar.gz: 9d87121dd840665b4bba91771b5f3e7f2a27e753d2dd374b5e461f1db172aa8d
 SHA512:
-  metadata.gz: 9250e85c200c5190a6f85fef44404fa0576d39621960af375b3cbd3cd7ffe172b19fff8021c4f32b1cd63ac453457c6c3bffd504abac9851091d3f4091530b45
-  data.tar.gz: 28194d92924dbac55258734f66ed3a4090ebe514d2edf8dcfd0368a244c4373111c70be23a32b1d3fb08f271f52ade9434529b4ea6dd91015bee8d0bfaa077b5
+  metadata.gz: 784b7e7894cfabc80c1146a9ac93ce7d466a872cb3d86caec664fd1f2c1fa453d125ca2528b03e1a6d55b71fc15f87fdd54eb05cebb32639365c144a7303777d
+  data.tar.gz: b22a989af8d7a2cfe867f6ad1485ebf88f312ba3eb7e0ea0df52663cd60a552cf1576c1c7fc85ee4caffb576397c2684d06f596e8bf993c6d6b3cdf610dcf837

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 ## [Unreleased]
 
+## [1.7.0]
+- Correct a row whose own `updated_at` predates a record embedded in its payload, then serialize again, behind `config.touch_records_behind_their_embedded_records` (default `false`). A serializer reads the parent's columns before the associations, so a child committed in between arrives under a timestamp older than itself, and the consumer discards the payload together with that child.
+
 ## [1.6.0]
 - Judge a payload's embedded children on their own timestamps instead of dropping them with the parent. When `persist_with_dionysus?` rejects a record, `persist` used to `next` past the whole loop body - the child `persist` calls included - so every `has_many` and `has_one` record embedded in that payload was discarded with it, with no error and no counter. The parent is still not written; only the traversal of its children is decoupled from its verdict. Each child re-enters `persist` and is guarded on its own `synced_updated_at || synced_created_at`, so a child the consumer has never seen is created and a child older than the row held locally is still skipped.
 - Reproduced at the consumer, not argued: `spec/embedded_children_persistence_spec.rb` runs a whole batch through the generated consumer with deduplication in place, from two failures observed in production. In the first, all seven money-bearing messages carried an `updated_at` 44 ms older than the zero-money payload already accepted, and the payment embedded in them - a row that did not exist locally - was never created. Three control examples pin the fixture down: the same shape at a timestamp the guard accepts persists the money, creates the embedded payment and calls `resolve_to_many_association`, so a red assertion is a statement about this gem rather than about the payload being malformed.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dionysus-rb (1.6.0)
+    dionysus-rb (1.7.0)
       activerecord (>= 5)
       activesupport (>= 3.2)
       concurrent-ruby

data/lib/dionysus/producer/config.rb

@@ -10,7 +10,8 @@ class Dionysus::Producer::Config
     :genesis_consistency_safety_delay, :hermes_event_producer, :publish_after_commit, :outbox_worker_publishing_delay,
     :high_priority_sidekiq_queue, :observers_inline_maximum_size, :remove_consecutive_duplicates_before_publishing,
     :include_serialized_at_in_payload, :publish_with_uncached_reads, :publish_consistent_snapshots,
-    :max_snapshot_attempts, :republish_deduplicated_records, :republish_deduplicated_records_delay
+    :max_snapshot_attempts, :republish_deduplicated_records, :republish_deduplicated_records_delay,
+    :touch_records_behind_their_embedded_records
 
   def self.default_sidekiq_queue
     :dionysus
@@ -98,6 +99,14 @@ class Dionysus::Producer::Config
     @observers_inline_maximum_size || 1000
   end
 
+  # Corrects a row whose own timestamp predates a record embedded in its payload, then serializes
+  # again. Off by default: it writes to the source table from the publish path.
+  def touch_records_behind_their_embedded_records
+    return @touch_records_behind_their_embedded_records if defined?(@touch_records_behind_their_embedded_records)
+
+    false
+  end
+
   # Off by default so it can be switched on per producer, and switched back off in one env change
   # if the extra reads ever cost more than they are worth.
   # How many times a payload may be re-serialized before the last attempt is published as it stands.

data/lib/dionysus/producer/embedded_timestamp_repair.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::EmbeddedTimestampRepair
+  TIMESTAMP_ATTRIBUTE = "updated_at"
+  PRIMARY_KEY_ATTRIBUTE = "id"
+
+  # Takes the block that serializes, so a repaired record can be serialized again in one place.
+  def self.call(records, config, &)
+    new(records, config).call(&)
+  end
+
+  attr_reader :records, :config
+  private :records, :config
+
+  def initialize(records, config)
+    @records = records
+    @config = config
+  end
+
+  def call
+    payload, read_at = yield
+    return [payload, read_at] unless config.touch_records_behind_their_embedded_records
+    return [payload, read_at] unless payload.is_a?(Array)
+    return [payload, read_at] unless any_repaired?(payload)
+
+    yield
+  end
+
+  private
+
+  # Pairing is positional, and the serializer is supplied by the application, so a payload that does
+  # not line up 1:1 could touch one record with another record's children. Fail closed instead.
+  # map, not any? - any? short-circuits and would leave the rest of a batch unrepaired
+  def any_repaired?(payload)
+    return false unless payload.size == Array(records).size
+
+    Array(records).zip(payload).map { |record, record_payload| repaired?(record, record_payload) }.any?
+  end
+
+  # A parent whose own timestamp predates a record embedded in it publishes a payload that describes
+  # no single moment, and the consumer discards it with its children. Correcting the row keeps the
+  # published timestamp equal to the row's own, which is what the republish path relies on.
+  def repaired?(record, record_payload)
+    return false unless repairable?(record, record_payload)
+
+    latest = embedded_timestamps(record_payload).max
+    return false unless latest && latest > record_payload[TIMESTAMP_ATTRIBUTE]
+
+    # WHERE guards the write: a stale read must never move the row backwards. When the row is already
+    # ahead nothing is written, and the reload alone repairs the payload.
+    updated = record.class.where(record.class.primary_key => record.id)
+      .where("#{record.class.quoted_table_name}.#{TIMESTAMP_ATTRIBUTE} < ?", latest)
+      .update_all(TIMESTAMP_ATTRIBUTE => latest)
+    # the row can be deleted between the guard and here, and raising would abort the whole batch
+    record.reload
+    instrument(record, updated)
+    true
+  rescue ActiveRecord::RecordNotFound
+    false
+  end
+
+  def repairable?(record, record_payload)
+    record.is_a?(ActiveRecord::Base) && record.persisted? &&
+      record_payload.is_a?(Hash) && record_payload[PRIMARY_KEY_ATTRIBUTE] == record.id &&
+      time_like?(record_payload[TIMESTAMP_ATTRIBUTE])
+  end
+
+  def embedded_timestamps(record_payload)
+    record_payload.each_value.flat_map do |value|
+      Array.wrap(value).filter_map do |embedded|
+        next unless embedded.is_a?(Hash)
+
+        timestamp = embedded[TIMESTAMP_ATTRIBUTE]
+        timestamp if time_like?(timestamp)
+      end
+    end
+  end
+
+  def instrument(record, updated)
+    config.instrumenter.increment("dionysus.publish.timestamp_repair",
+      tags: ["model:#{record.class}", "outcome:#{updated.zero? ? "stale_read" : "row_behind"}"])
+  end
+
+  def time_like?(value)
+    value.respond_to?(:acts_like_time?) && value.acts_like_time?
+  end
+end

data/lib/dionysus/producer/karafka_responder_generator.rb

@@ -48,7 +48,8 @@ class Dionysus::Producer::KarafkaResponderGenerator
 
               # consumers rank duplicates by this stamp, so it has to describe the attempt actually
               # published - with retries that is the last one, not the first
-              payload, read_at = serialize_consistently(records, topic, batch_options)
+              payload, read_at = Dionysus::Producer::EmbeddedTimestampRepair
+                .call(records, config) { serialize_consistently(records, topic, batch_options) }
               serialized_at = config.include_serialized_at_in_payload ? read_at : nil
 
               event_payload = {

data/lib/dionysus/version.rb

@@ -3,5 +3,5 @@
 module Dionysus
   module Version
   end
-  VERSION = "1.6.0"
+  VERSION = "1.7.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dionysus-rb
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.7.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-01 00:00:00.000000000 Z
+date: 2026-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -500,6 +500,7 @@ files:
 - lib/dionysus/producer/base_responder.rb
 - lib/dionysus/producer/config.rb
 - lib/dionysus/producer/deleted_record_serializer.rb
+- lib/dionysus/producer/embedded_timestamp_repair.rb
 - lib/dionysus/producer/genesis.rb
 - lib/dionysus/producer/genesis/performed.rb
 - lib/dionysus/producer/genesis/stream_job.rb