dionysus-rb 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0934079a452c75d517f1954e9cc6650c4b6219273c329013b676f336ea0d9e13'
-  data.tar.gz: bee752d4720e8f4249867ecf3506b725fa9ba9cd678ebadf9ac0ac893c635925
+  metadata.gz: 64406a4ca852cfc6d663586730005d9c562121571cb0b876de7584eb63b2c49f
+  data.tar.gz: c30a51ca9f590795e55021209f274d67a18aa197bd2ebb7be3ed471940ee7f6a
 SHA512:
-  metadata.gz: e12c164f5909f35e9b0d4336fb31ebb3d79e064409243beaad6e40c32f5590ff296ee805601de312b111fcc5fbb10d86008064cb2c229de56d7eab8518e6c9f9
-  data.tar.gz: 6a0fffb5d3b4d65eab34a0aaa16871002b8713addb45e07a1094706559fc978d5b37c885d6d2179c481bea436eddfe9805db0385a464ddb6a70b056ccf10e489
+  metadata.gz: bd48b03ee34b15dd0481733a9bdd726ef23c871d2a6a19249501ffa5ebb200559a2c301785a0e9449e79059eb98f3166da2c70049d087003dc2684c89ab0a013
+  data.tar.gz: 13f6d2e1068ea38a8c8954e594fe36382d8d6a061228c5f1f9c5de722651c495363be8569874757ce8f1fe41232495597676532d7e56da1a5a3fe25a4c0a32f1

data/.rubocop_todo.yml

@@ -12,6 +12,7 @@ Layout/LineLength:
     - 'lib/dionysus.rb'
     - 'lib/dionysus/consumer/persistor.rb'
     - 'lib/dionysus/producer/genesis/streamer/standard_job.rb'
+    - 'lib/dionysus/consumer/batch_events_publisher.rb'
     - 'spec/dionysus/consumer/karafka_consumer_generator_spec.rb'
     - 'spec/dionysus/producer/outbox/publisher_spec.rb'
     - 'spec/dionysus/producer/genesis_spec.rb'

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 ## [Unreleased]
 
+## [0.5.0]
+- Publish consumed messages batch as `dionysus.consume_batch` event.
+
 ## [0.4.0]
 - Allow `Dionysus::Producer::Genesis::StreamJob`/`Dionysus::Producer::Genesis::Streamer` to take more options and perform filtering by extra conditions. This is useful if you only need to stream some of the records.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dionysus-rb (0.4.0)
+    dionysus-rb (0.5.0)
       activerecord (>= 5)
       activesupport (>= 3.2)
       concurrent-ruby

data/README.md

@@ -194,7 +194,7 @@ Dionysus::Producer.responders_for(Record).each do |responder|
   message = [].tap do |current_message|
     current_message << ["record_created", created_records.to_a, {}]
     if canceled_records.any?
-      current_message << ["record_destroyed", canceled_records.map { |record| RecordPrometheusDTO.new(record) }, { serialize: false }]
+      current_message << ["record_destroyed", canceled_records.map { |record| RecordDionysusDTO.new(record) }, { serialize: false }]
     end
   end
 
@@ -1137,7 +1137,7 @@ Dionysus::Consumer.configure do |config|
   config.resolve_synced_data_hash_proc = ->(record) { record.synced_data_model.synced_data_hash } # optional, defaults to ->(record) { record.public_send(Dionysus::Consumer.configuration.synced_data_attribute).to_h }
   config.sidekiq_queue = :default # optional, defaults to `:dionysus`
   config.message_filter = FilterIgnoringLargeMessageToAvoidOutofMemoryErrors.new(error_handler: Sentry) #  DEPRECATED - not required, defaults to Dionysus::Utils::DefaultMessageFilter, which doesn't ignore any messages. It can be useful when you want to ignore some messages, e.g. some very large ones that would cause OOM error. Check the implementation of `Dionysus::Utils::DefaultMessageFilter for more details to understand what kind of arguments are available to set the condition. `error_handler` needs to implement Sentry-like interface. Kept for backwards compatibility, please use `message_filters` instead.
-  config.message_filters = [FilterIgnoringLargeMessageToAvoidOutofMemoryErrors.new(error_handler: Sentry)] # not required, defaults to [Dionysus::Utils::DefaultMessageFilter], which doesn't ignore any messages. It can be useful when you want to ignore some messages, e.g. some very large ones that would cause OOM error. Check the implementation of `Dionysus::Utils::DefaultMessageFilter for more details to understand what kind of arguments are available to set the condition. `error_handler` needs to implement Sentry-like interface. 
+  config.message_filters = [FilterIgnoringLargeMessageToAvoidOutofMemoryErrors.new(error_handler: Sentry)] # not required, defaults to [Dionysus::Utils::DefaultMessageFilter], which doesn't ignore any messages. It can be useful when you want to ignore some messages, e.g. some very large ones that would cause OOM error. Check the implementation of `Dionysus::Utils::DefaultMessageFilter for more details to understand what kind of arguments are available to set the condition. `error_handler` needs to implement Sentry-like interface.
 
   # if you ever need to provide mapping:
 
@@ -1154,7 +1154,9 @@ end
 
 Check publisher for reference about instrumentation and event bus. The only difference is about the methods that are instrumented and events that are published.
 
-For the event bus, you may expect the `dionysus.consume` event. It contains the following attributes:
+For the event bus, you may expect two events:
+
+1. `dionysus.consume` event. It contains the following attributes:
 - `topic_name`, e.g. "v3_inbox", "v3_rentals"
 - `model_name`, e.g. "Conversation", "Rental"
 - `event_name`, e.g. "rental_created", "converation_updated", "message_destroyed"
@@ -1168,6 +1170,8 @@ For the event bus, you may expect the `dionysus.consume` event. It contains the
 }
 ```
 
+2. `dionysus.consume_batch` event. It's essentially an array of events available under `dionysus.consume` event representing an entire Karafka consumers messages' batch.
+
 Event bus is the recommended way to do something upon consuming events if you want to avoid putting that logic into ActiveRecord callbacks.
 
 

data/lib/dionysus/consumer/batch_events_publisher.rb

@@ -11,8 +11,10 @@ class Dionysus::Consumer::BatchEventsPublisher
 
   def publish(processed_events)
     processed_events
-      .map(&method(:to_event_data))
-      .each(&method(:publish_via_event_bus))
+      .map { |dionysus_event| to_event_data(dionysus_event) }
+      .select(&:present?) # potentially necessary for some Kafka versions? couldn't reproduce this reliably, but the integration spec added a tombstone message for `publish_events_batch_via_event_bus` in the older version
+      .then { |mapped_events| publish_events_batch_via_event_bus(mapped_events) }
+      .then { |mapped_events| mapped_events.each { |event_data| publish_single_event_via_event_bus(event_data) } }
   end
 
   private
@@ -27,7 +29,12 @@ class Dionysus::Consumer::BatchEventsPublisher
     }
   end
 
-  def publish_via_event_bus(event_data)
+  def publish_events_batch_via_event_bus(events_batch_data)
+    config.event_bus.publish("dionysus.consume_batch", events_batch_data)
+    events_batch_data
+  end
+
+  def publish_single_event_via_event_bus(event_data)
     config.event_bus.publish("dionysus.consume", event_data)
   end
 end

data/lib/dionysus/version.rb

@@ -3,5 +3,5 @@
 module Dionysus
   module Version
   end
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dionysus-rb
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-23 00:00:00.000000000 Z
+date: 2024-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord