dionysus-rb 0.5.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64406a4ca852cfc6d663586730005d9c562121571cb0b876de7584eb63b2c49f
-  data.tar.gz: c30a51ca9f590795e55021209f274d67a18aa197bd2ebb7be3ed471940ee7f6a
+  metadata.gz: d190bbc3fd4b2ca0aa7c7d1be7c0ddaa6a3b6bdcbd6556b90e4752c2e875bfc6
+  data.tar.gz: 33379f5ccc97bceee7caad1d9b3383e84c3073e48b15bda1d0f592e765c39352
 SHA512:
-  metadata.gz: bd48b03ee34b15dd0481733a9bdd726ef23c871d2a6a19249501ffa5ebb200559a2c301785a0e9449e79059eb98f3166da2c70049d087003dc2684c89ab0a013
-  data.tar.gz: 13f6d2e1068ea38a8c8954e594fe36382d8d6a061228c5f1f9c5de722651c495363be8569874757ce8f1fe41232495597676532d7e56da1a5a3fe25a4c0a32f1
+  metadata.gz: cafb4cb76ad00f5dbbc45f9c3a578284ef554aaf680310286961d1f8251620bc67c10e190f301ac06fca1c8e55be9fab2e75977cc85ecf80f21503504558da20
+  data.tar.gz: 78e8c07d652f3dcc46cb3a527834cb72b2290b119266e4362cde3ddbf1d5679c147cb03cce34acb6682451c6453fb58532faab8a7bf5be43a6db6be791e7e1e2

data/.circleci/config.yml

@@ -4,7 +4,7 @@ jobs:
     working_directory: ~/BookingSync/dionysus-rb
     parallelism: 1
     docker:
-      - image: cimg/ruby:2.7.6
+      - image: cimg/ruby:3.3.11
         environment:
           POSTGRES_URL: "postgres://docker:docker@127.0.0.1:5432"
           DATABASE_URL: "postgres://docker:docker@127.0.0.1:5432/circle_test"

data/.github/workflows/ci.yml

@@ -6,10 +6,10 @@ jobs:
       fail-fast: true
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.7
+          ruby-version: '3.3'
           bundler-cache: true
       - run: bundle exec rubocop
   rspec:
@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ['2.7', '3.0.6', '3.1', '3.2']
+        ruby: ['3.3', '3.4']
     runs-on: ubuntu-latest
     env:
       DD_PROFILING_NO_EXTENSION: true
@@ -27,23 +27,6 @@ jobs:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: postgres
       POSTGRES_DB: dionysus-rb-test
-      CLUSTER_ID: kafka-docker-cluster-1
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_PROCESS_ROLES: broker,controller
-      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
-      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
-      KAFKA_BROKER_ID: 1
-      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
-      ALLOW_PLAINTEXT_LISTENER: 'yes'
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
-      KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
-      KAFKA_CREATE_TOPICS: "v102_rentals:1:1"
     services:
       postgres:
         image: postgres:14
@@ -63,14 +46,34 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+      kafka:
+        image: confluentinc/cp-kafka:7.5.2
+        env:
+          CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
+          KAFKA_NODE_ID: 1
+          KAFKA_PROCESS_ROLES: broker,controller
+          KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
+          KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
+          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+          KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+          KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+          KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+          KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+          KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+          KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+        ports:
+          - 9092:9092
+        options: >-
+          --health-cmd "kafka-topics --bootstrap-server localhost:9092 --list"
+          --health-interval 10s
+          --health-timeout 10s
+          --health-retries 12
+          --health-start-period 20s
     steps:
-    - uses: actions/checkout@v2
-    - name: Start Kafka with docker-compose
-      run: |
-        docker-compose up -d || (sleep 5 && docker-compose up -d)
-
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
-
       with:
         ruby-version: ${{ matrix.ruby }}
         bundler-cache: true

data/.rubocop.yml

@@ -1,6 +1,6 @@
 inherit_from: .rubocop_todo.yml
 
-require:
+plugins:
   - rubocop-performance
   - rubocop-rspec
   - rubocop-rake
@@ -11,7 +11,7 @@ inherit_mode:
 
 AllCops:
   NewCops: enable
-  TargetRubyVersion: 2.7
+  TargetRubyVersion: 3.3
   Exclude:
     - "db/**/*"
     - "bin/**/*"
@@ -129,6 +129,21 @@ RSpec/ExampleLength:
 Naming/VariableNumber:
   Enabled: false
 
+# Same rationale as Naming/VariableNumber: the specs deliberately number fixtures
+# (rental_1, metadata_2) when they describe an ordered batch.
+RSpec/IndexedLet:
+  Enabled: false
+
+# Development dependencies are declared in the gemspec, not in the Gemfile.
+Gemspec/DevelopmentDependencies:
+  Enabled: false
+
+# `is_expected_block` (spec/support/is_expected_block.rb) is `expect { subject }`,
+# which the cop cannot see through.
+RSpec/NoExpectationExample:
+  AllowedPatterns:
+    - "^is_expected_block"
+
 Metrics/BlockLength:
     Max: 30
     Exclude:
@@ -136,7 +151,7 @@ Metrics/BlockLength:
       - 'lib/dionysus/support/rspec/outbox_publishable.rb'
       - 'lib/dionysus/consumer/persistor.rb'
 
-Naming/PredicateName:
+Naming/PredicatePrefix:
   Enabled: false
 
 Metrics/AbcSize:

data/.rubocop_todo.yml

@@ -52,3 +52,17 @@ Metrics/CyclomaticComplexity:
 Metrics/BlockLength:
   Exclude:
     - 'lib/dionysus/producer/karafka_responder_generator.rb'
+
+# `declared_attributes` and friends are Arrays of [name, options] pairs, not Hashes,
+# so the cop's each_key/each_value suggestion raises NoMethodError.
+Style/HashEachMethods:
+  Exclude:
+    - 'lib/dionysus/producer/model_serializer.rb'
+
+# integration_spec prints progress for a long-running Kafka run; the null provider
+# specs use `puts` as the sentinel proving the block was called.
+RSpec/Output:
+  Exclude:
+    - 'spec/dionysus/utils/null_retry_provider_spec.rb'
+    - 'spec/dionysus/utils/null_transaction_provider_spec.rb'
+    - 'spec/integration_spec.rb'

data/.ruby-version

@@ -0,0 +1 @@
+ruby-3.3.11

data/CHANGELOG.md

@@ -1,5 +1,51 @@
 ## [Unreleased]
 
+## [1.1.0]
+
+### Fixed
+
+- Keep the last published message when removing duplicates from a consumer batch. `RemoveDuplicatesStrategy` kept the message with the highest payload `updated_at` per event and id, but payloads are serialized at publish time, so freshness follows publish order rather than `updated_at`: a payload carrying values computed on read from associated records changes without the record's own `updated_at` changing, and can then arrive with an older `updated_at` than a staler message. Those messages were silently discarded, losing the change in every consumer with no error and no trace. The message with the highest Kafka offset is kept instead - a Karafka batch comes from a single topic-partition, so offsets strictly increase in publish order.
+
+## [1.0.0]
+
+Brings `dionysus-rb` back in line with the private gem it was extracted from, which had moved on
+by four releases.
+
+### Breaking changes
+
+- Require Ruby >= 3.3 (was >= 2.7).
+- Require karafka `~> 2.4` (was `~> 2.0`). Follow https://karafka.io/docs/Upgrades-2.4/ - most
+  notably, `Karafka::Messages::Metadata` no longer carries a single `deserializer`, it carries a
+  `deserializers` config with separate `payload` and `key` entries, and `key` is now `raw_key`.
+  This only affects code that constructs metadata by hand, such as consumer specs.
+- The instrumenter is now expected to respond to `increment(name, options)` in addition to
+  `instrument`. `Dionysus::Utils::NullInstrumenter` implements it; a custom instrumenter that does
+  not will raise when a publish produces no message (see below).
+
+### Added
+
+- Notify when publishing an outbox record produces no Kafka message at all.
+  `Dionysus::Producer::Outbox::Publisher#publish` could complete successfully having sent nothing -
+  a model declared only as a `with:` dependency has no top-level responder, so when its parent
+  could not be resolved the call returned normally - and `RecordsProcessor` then marked the record
+  as published, losing the change in every consumer with no error and no retry. Such a record is
+  now logged, counted as `dionysus.publish.nothing_published` and reported through the configured
+  `error_handler`. It notifies rather than raises: the producer cannot distinguish a legitimately
+  deleted parent from one that is not visible yet.
+- `Dionysus.initialize_application!` accepts `consumer_group_name`, which is used verbatim as the
+  consumer group name and takes precedence over `consumer_group_prefix`. `consumer_group_prefix`
+  stays supported and unchanged, so the documented migration path from `bookingsync-prometheus`
+  keeps working.
+
+### Fixed
+
+- Compatibility with the `datadog` gem 2.0, which renamed the `span_type:` keyword of
+  `Datadog::Tracing#trace` to `type:`. `Dionysus::Producer::Outbox::DatadogTracer` now picks the
+  right keyword depending on whether the pre-2.0 `ddtrace` gem is loaded.
+- Do not publish the `dionysus.consume_batch` event when the batch is empty.
+- `Dionysus::Producer::Outbox::Publisher` no longer trips over a `nil` entry in a parent
+  association collection.
+
 ## [0.5.0]
 - Publish consumed messages batch as `dionysus.consume_batch` event.
 

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    dionysus-rb (0.5.0)
+    dionysus-rb (1.1.0)
       activerecord (>= 5)
       activesupport (>= 3.2)
       concurrent-ruby
       dry-monitor
       file-based-healthcheck
-      karafka (~> 2.0)
+      karafka (~> 2.4)
       sidekiq
       sidekiq-cron
       sigurd
@@ -34,8 +34,8 @@ GEM
       mutex_m
       tzinfo (~> 2.0)
     amq-protocol (2.3.2)
-    ast (2.4.2)
-    base64 (0.1.1)
+    ast (2.4.3)
+    base64 (0.3.0)
     bigdecimal (3.1.4)
     bunny (2.22.0)
       amq-protocol (~> 2.3, >= 2.3.1)
@@ -43,20 +43,20 @@ GEM
     byebug (11.1.3)
     carrot-top (0.0.7)
       json
+    cgi (0.5.2)
     concurrent-ruby (1.2.2)
     connection_pool (2.4.1)
     crypt_keeper (0.13.1)
       activerecord (>= 3.0)
       activesupport (>= 3.0)
-    datadog-ci (0.5.1)
+    datadog (2.40.0)
+      cgi
+      datadog-ruby_core_source (~> 3.5, >= 3.5.4)
+      libdatadog (~> 36.0.0.1.0)
+      libddwaf (~> 1.30.0.0.0)
+      logger
       msgpack
-    ddtrace (1.18.0)
-      datadog-ci (~> 0.5.0)
-      debase-ruby_core_source (= 3.2.3)
-      libdatadog (~> 5.0.0.1.0)
-      libddwaf (~> 1.14.0.0.0)
-      msgpack
-    debase-ruby_core_source (3.2.3)
+    datadog-ruby_core_source (3.5.4)
     diff-lcs (1.5.0)
     dogstatsd-ruby (5.5.0)
     drb (2.1.1)
@@ -118,32 +118,41 @@ GEM
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
-    json (2.6.2)
-    karafka (2.2.7)
-      karafka-core (>= 2.2.2, < 2.3.0)
-      thor (>= 0.20)
-      waterdrop (>= 2.6.6, < 3.0.0)
+    json (2.21.2)
+    karafka (2.5.2)
+      base64 (~> 0.2)
+      karafka-core (>= 2.5.6, < 2.6.0)
+      karafka-rdkafka (>= 0.22.0)
+      waterdrop (>= 2.8.9, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.2.2)
-      concurrent-ruby (>= 1.1)
-      karafka-rdkafka (>= 0.13.1, < 0.14.0)
-    karafka-rdkafka (0.13.5)
+    karafka-core (2.5.13)
+      karafka-rdkafka (>= 0.20.0)
+      logger (>= 1.6.0)
+    karafka-rdkafka (0.22.2)
       ffi (~> 1.15)
+      json (> 2.0)
+      logger
       mini_portile2 (~> 2.6)
       rake (> 12)
-    libdatadog (5.0.0.1.0)
-    libddwaf (1.14.0.0.0)
+    language_server-protocol (3.17.0.6)
+    libdatadog (36.0.0.1.0)
+    libddwaf (1.30.0.0.2)
       ffi (~> 1.0)
+    lint_roller (1.1.0)
+    logger (1.7.0)
     mini_portile2 (2.8.4)
     minitest (5.20.0)
-    msgpack (1.7.2)
+    msgpack (1.8.4)
     multi_json (1.15.0)
     mutex_m (0.1.2)
-    parallel (1.22.1)
-    parser (3.1.2.1)
+    parallel (2.1.0)
+    parser (3.3.12.0)
       ast (~> 2.4.1)
+      racc
     pg (1.3.5)
+    prism (1.9.0)
     raabro (1.4.0)
+    racc (1.8.1)
     rack (2.2.4)
     rainbow (3.1.1)
     rake (13.0.6)
@@ -153,10 +162,9 @@ GEM
       redis (>= 3.0.4)
     redlock (1.2.2)
       redis (>= 3.0.0, < 5.0)
-    regexp_parser (2.5.0)
+    regexp_parser (2.12.0)
     request_store (1.5.1)
       rack (>= 1.4)
-    rexml (3.2.5)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
@@ -175,26 +183,32 @@ GEM
       rspec-core (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
     rspec-support (3.12.1)
-    rubocop (1.35.0)
+    rubocop (1.89.0)
       json (~> 2.3)
-      parallel (~> 1.10)
-      parser (>= 3.1.2.1)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (>= 1.10)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.20.1, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.21.0)
-      parser (>= 3.1.1.0)
-    rubocop-performance (1.14.3)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
-    rubocop-rake (0.6.0)
-      rubocop (~> 1.0)
-    rubocop-rspec (2.12.1)
-      rubocop (~> 1.31)
-    ruby-progressbar (1.11.0)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.50.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.7)
+    rubocop-performance (1.26.1)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.75.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
+    rubocop-rake (0.7.1)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.72.1)
+    rubocop-rspec (3.10.2)
+      lint_roller (~> 1.1)
+      regexp_parser (>= 2.0)
+      rubocop (~> 1.86, >= 1.86.2)
+    ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     sentry-ruby (5.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -215,14 +229,16 @@ GEM
     sorted_set (1.0.3)
       rbtree
       set (~> 1.0)
-    thor (1.2.2)
     timecop (0.9.5)
     timeout (0.4.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.2.0)
-    waterdrop (2.6.7)
-      karafka-core (>= 2.1.1, < 3.0.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.2.0)
+    waterdrop (2.8.13)
+      karafka-core (>= 2.4.9, < 3.0.0)
+      karafka-rdkafka (>= 0.20.0)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.12)
 
@@ -234,7 +250,7 @@ DEPENDENCIES
   bundler
   byebug
   crypt_keeper
-  ddtrace
+  datadog
   dionysus-rb!
   dogstatsd-ruby
   hermes-rb

data/README.md

@@ -62,7 +62,19 @@ Dionysus.initialize_application!(
 )
 ```
 
-By default, the name of the consumer grpup will be  "NAME_OF_THE_APP_dionysus_consumer_group_for_NAME_OF_THE_APP" where `dionysus_consumer_group_for` is a `consumer_group_prefix`.
+By default, the name of the consumer group will be `"dionysus_consumer_group_for_NAME_OF_THE_APP"`, where `dionysus_consumer_group_for` is the `consumer_group_prefix`.
+
+If a prefix is not enough and you need to control the entire name, provide `consumer_group_name`, which is used verbatim and takes precedence over `consumer_group_prefix`:
+
+``` rb
+Dionysus.initialize_application!(
+  environment: ENV["RAILS_ENV"],
+  seed_brokers: ENV.fetch("DIONYSUS_SEED_BROKER").split(";"),
+  client_id: "NAME_OF_THE_APP",
+  logger: Rails.logger,
+  consumer_group_name: "your_custom_consumer_group_name"
+)
+```
 
 
 And define `dionysus.rb` initializer with your Kafka topics:
@@ -234,9 +246,16 @@ class Dionysus::Utils::NullInstrumenter
   def self.instrument(name, payload = {})
     yield
   end
+
+  def self.increment(name, options = {})
+  end
 end
 ```
 
+`increment` is used for counters that are not tied to a block, currently only
+`dionysus.publish.nothing_published` (see the Transactional Outbox Pattern section). A custom
+instrumenter that does not implement it will raise when that counter is emitted.
+
 
 Event Bus is useful if you want to react to some events, with the following interface ((this is the default class)):
 
@@ -508,6 +527,24 @@ bundle exec outbox_worker_health_check
 
 It works for both readiness and liveness checks.
 
+##### Publishes that produce no Kafka message
+
+`Dionysus::Producer::Outbox::Publisher#publish` can legitimately match no responder at all - a
+model declared only as a `with:` dependency has no top-level responder of its own, so when its
+parent cannot be resolved nothing gets sent. The outbox record would still be stamped as
+published, which means the change is silently lost in every consumer.
+
+Such a record is now logged via `Dionysus.logger.error`, reported through the configured
+`error_handler` with `capture_message`, and counted as:
+
+```
+instrumenter.increment("dionysus.publish.nothing_published", tags: ["resource:<class>", "topic:<topic>"])
+```
+
+It notifies rather than raises, because the producer cannot tell a legitimately deleted parent
+from one that is merely not visible yet, and raising on the former would retry forever. Reprocess
+the reported record from the console.
+
 #### Tombstoning records
 
 The only way to get rid of messages under a given key from Kafka is to tombstone them. Use `Dionysus::Producer::Outbox::TombstonePublisher` to do it:
@@ -677,7 +714,19 @@ Dionysus.initialize_application!(
 )
 ```
 
-By default, the name of the consumer grpup will be  "NAME_OF_THE_APP_dionysus_consumer_group_for_NAME_OF_THE_APP" where `dionysus_consumer_group_for` is a `consumer_group_prefix`.
+By default, the name of the consumer group will be `"dionysus_consumer_group_for_NAME_OF_THE_APP"`, where `dionysus_consumer_group_for` is the `consumer_group_prefix`.
+
+If a prefix is not enough and you need to control the entire name, provide `consumer_group_name`, which is used verbatim and takes precedence over `consumer_group_prefix`:
+
+``` rb
+Dionysus.initialize_application!(
+  environment: ENV["RAILS_ENV"],
+  seed_brokers: ENV.fetch("DIONYSUS_SEED_BROKER").split(";"),
+  client_id: "NAME_OF_THE_APP",
+  logger: Rails.logger,
+  consumer_group_name: "your_custom_consumer_group_name"
+)
+```
 
 
 And define `dionysus.rb` initializer:

data/dionysus-rb.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "byebug"
   spec.add_development_dependency "crypt_keeper"
-  spec.add_development_dependency "ddtrace"
+  spec.add_development_dependency "datadog"
   spec.add_development_dependency "dogstatsd-ruby"
   spec.add_development_dependency "hermes-rb"
   spec.add_development_dependency "pg"
@@ -52,13 +52,13 @@ Gem::Specification.new do |spec|
   spec.add_dependency "concurrent-ruby"
   spec.add_dependency "dry-monitor"
   spec.add_dependency "file-based-healthcheck"
-  spec.add_dependency "karafka", "~> 2.0"
+  spec.add_dependency "karafka", "~> 2.4"
   spec.add_dependency "sidekiq"
   spec.add_dependency "sidekiq-cron"
   spec.add_dependency "sigurd"
   spec.add_dependency "waterdrop", "~> 2.0"
   spec.add_dependency "zeitwerk"
 
-  spec.required_ruby_version = ">= 2.7"
+  spec.required_ruby_version = ">= 3.3"
   spec.metadata["rubygems_mfa_required"] = "true"
 end

data/lib/dionysus/consumer/batch_events_publisher.rb

@@ -30,7 +30,7 @@ class Dionysus::Consumer::BatchEventsPublisher
   end
 
   def publish_events_batch_via_event_bus(events_batch_data)
-    config.event_bus.publish("dionysus.consume_batch", events_batch_data)
+    config.event_bus.publish("dionysus.consume_batch", events_batch_data) if events_batch_data.any?
     events_batch_data
   end
 

data/lib/dionysus/consumer/params_batch_processor.rb

@@ -57,13 +57,13 @@ class Dionysus::Consumer::ParamsBatchProcessor
     message_filters.find { |f| f.ignore_message?(topic: topic, message: message, transformed_data: transformed_data) }
   end
 
-  def instrument(label, options = {}, &block)
-    config.instrumenter.instrument(label, options, &block)
+  def instrument(label, options = {}, &)
+    config.instrumenter.instrument(label, options, &)
   end
 
-  def with_mutex(metadata, config, &block)
+  def with_mutex(metadata, config, &)
     message_key = metadata.key || SecureRandom.uuid
     config.processing_mutex_provider.send(config.processing_mutex_method_name, "Dionysus-#{message_key}",
-      &block)
+      &)
   end
 end

data/lib/dionysus/consumer/params_batch_transformations/remove_duplicates_strategy.rb

@@ -10,18 +10,16 @@ class Dionysus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy
   private
 
   # the idea is following:
-  # 1. group messages by event and id - for a given model we can expect unique messages for _created and _deleted ecents
+  # 1. group messages by event and id - for a given model we can expect unique messages for _created and _deleted events
   #   but we can have multiple _updated events, so this is where we are interested in removing duplicates
-  # 2. sort each group by updated_at, reverse (because the sort is ascending) and take the first one -
-  #   this way we will have the most recent update
+  # 2. from each group keep the message published last (highest offset) - payloads are serialized at
+  #   publish time, so that is the freshest state; "updated_at" does not track it
   # 3. flatten the array of arrays as all groups will have a single item
-  # It is safer to apply sorting just for the _updated events to a given model as otherwise we could change the order of
-  #   the messages for a different type which might be not desirable
   def transform_messages_array(params_batch)
     params_batch
       .to_a
       .group_by { |batch| grouping_key_by_event_and_id(batch) }
-      .map { |_, group| group.max_by { |batch| updated_at_from_batch(batch) } }
+      .map { |_, group| group.max_by(&:offset) }
       .flatten
   end
 
@@ -32,14 +30,6 @@ class Dionysus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy
     ].join
   end
 
-  def updated_at_from_batch(batch)
-    timestamp = batch.payload.fetch("message", []).first.to_h.fetch("data", []).first.to_h.fetch("updated_at", nil)
-
-    return timestamp.to_datetime if timestamp.respond_to?(:to_datetime)
-
-    Time.current
-  end
-
   def duplicates_removal_not_applicable?(params_batch)
     any_message_containing_more_than_one_event?(params_batch) || any_event_containing_more_than_one_item(params_batch)
   end

data/lib/dionysus/consumer/persistor.rb

@@ -151,7 +151,7 @@ class Dionysus::Consumer::Persistor
     end
   end
 
-  def instrument(label, options = {}, &block)
-    config.instrumenter.instrument(label, options, &block)
+  def instrument(label, options = {}, &)
+    config.instrumenter.instrument(label, options, &)
   end
 end

data/lib/dionysus/consumer/registry.rb

@@ -8,9 +8,9 @@ class Dionysus::Consumer::Registry
     @container = {}
   end
 
-  def namespace(namespace, &block)
+  def namespace(namespace, &)
     registration = Registration.new(namespace)
-    registration.instance_eval(&block)
+    registration.instance_eval(&)
     container[namespace] = registration
   end
 
@@ -32,8 +32,8 @@ class Dionysus::Consumer::Registry
       @deserializer_klass = deserializer_klass
     end
 
-    def topic(name, options = {}, &block)
-      new_topic = Topic.new(namespace, name, deserializer_klass, options, &block)
+    def topic(name, options = {}, &)
+      new_topic = Topic.new(namespace, name, deserializer_klass, options, &)
       consumer = Dionysus::Consumer::KarafkaConsumerGenerator.new.generate(
         Dionysus::Consumer.configuration, new_topic
       )

data/lib/dionysus/consumer/synchronizable_model.rb

@@ -27,7 +27,7 @@ class Dionysus::Consumer::SynchronizableModel < SimpleDelegator
 
   def assign_attributes_from_dionysus(attributes)
     public_send("#{synced_data_attribute}=", attributes)
-    reverse_mapping = config.attributes_mapping_for_model(model.model_name).to_a.map(&:reverse).to_h
+    reverse_mapping = config.attributes_mapping_for_model(model.model_name).to_a.to_h(&:reverse)
 
     assignable_attributes = extract_assignable_attributes(attributes)
       .map { |key, value| apply_mapping(reverse_mapping, key, value) }

data/lib/dionysus/consumer.rb

@@ -13,9 +13,9 @@ class Dionysus::Consumer
     configuration.registry
   end
 
-  def self.declare(&config)
+  def self.declare(&)
     registry = Dionysus::Consumer::Registry.new
-    registry.instance_eval(&config)
+    registry.instance_eval(&)
 
     Dionysus.inject_routing!(registry)
 

data/lib/dionysus/producer/base_responder.rb

@@ -7,7 +7,7 @@ class Dionysus::Producer::BaseResponder
 
     def topic(topic_name)
       self.topics ||= {}
-      self.topics[topic_name] = topic_name.to_s
+      topics[topic_name] = topic_name.to_s
     end
 
     def call(*data)

data/lib/dionysus/producer/outbox/datadog_tracer.rb

@@ -9,7 +9,8 @@ class Dionysus::Producer::Outbox::DatadogTracer
   end
 
   def trace(event_name, topic)
-    tracer.trace(event_name, span_type: "worker", service: self.class.service_name, on_error: error_handler) do |span|
+    tracer.trace(event_name, span_type_key => "worker", service: self.class.service_name,
+      on_error: error_handler) do |span|
       span.set_tag("topic", topic)
 
       yield
@@ -26,6 +27,16 @@ class Dionysus::Producer::Outbox::DatadogTracer
     end
   end
 
+  # datadog 2.0 renamed the `span_type:` keyword of Datadog::Tracing#trace to `type:`.
+  # DDTrace is only defined by the pre-2.0 ddtrace gem.
+  def span_type_key
+    if defined?(DDTrace)
+      :span_type
+    else
+      :type
+    end
+  end
+
   def error_handler
     ->(span, error) { span.set_error(error) }
   end

data/lib/dionysus/producer/outbox/model.rb

@@ -78,7 +78,7 @@ module Dionysus::Producer::Outbox::Model
   end
 
   def error
-    if error_class_arity == 1 || error_class_arity == -1
+    if [1, -1].include?(error_class_arity)
       error_class.constantize.new(error_message)
     else
       StandardError.new("#{error_class_constant}: #{error_message}")

data/lib/dionysus/producer/outbox/publishable.rb

@@ -74,8 +74,7 @@ class Dionysus::Producer::Outbox::Publishable < SimpleDelegator
       .map(&:primary_topic)
     topics_from_dependencies = Dionysus::Producer
       .responders_for_dependency_parent(model.class)
-      .map(&:last)
-      .map(&:primary_topic)
+      .map { |dependency| dependency.last.primary_topic }
 
     [top_level_topics, topics_from_dependencies]
       .flatten

data/lib/dionysus/producer/outbox/publisher.rb

@@ -33,8 +33,10 @@ class Dionysus::Producer::Outbox::Publisher
         return
       end
 
-      publish_for_top_level_resource(outbox_record, resource, event_name, topic, options)
-      publish_for_dependency(resource, topic, options)
+      published_count = publish_for_top_level_resource(outbox_record, resource, event_name, topic, options) +
+        publish_for_dependency(resource, topic, options)
+      handle_nothing_published(resource, event_name, topic) if published_count.zero?
+      published_count
     end
   end
 
@@ -67,19 +69,20 @@ class Dionysus::Producer::Outbox::Publisher
   delegate :logger, to: Dionysus
 
   def publish_for_top_level_resource(outbox_record, resource, event_name, topic, options)
-    Dionysus::Producer.responders_for_model_for_topic(resource.class, topic).each do |responder|
+    Dionysus::Producer.responders_for_model_for_topic(resource.class, topic).sum do |responder|
       partition_key = outbox_record.partition_key.presence || Dionysus::Producer::PartitionKey.new(
         resource
       ).to_key(responder: responder)
       key = Dionysus::Producer::Key.new(resource).to_key
 
       responder.call(generate_message(event_name, resource), options.merge(partition_key: partition_key, key: key))
+      1
     end
   end
 
   def publish_for_dependency(resource, topic, options)
     Dionysus::Producer.responders_for_dependency_parent_for_topic(resource.class,
-      topic).each do |parent_klass, responder|
+      topic).sum do |parent_klass, responder|
       parent_event_name = Dionysus::Producer::Outbox::EventName.new(
         parent_klass.model_name.singular
       ).updated
@@ -89,10 +92,11 @@ class Dionysus::Producer::Outbox::Publisher
       elsif resource.class.reflect_on_association(parent_klass.model_name.plural)
         parent_records = resource.public_send(parent_klass.model_name.plural)
       else
-        next
+        next 0
       end
 
-      example_parent_record = parent_records.first or next
+      parent_records = parent_records.to_a.compact
+      example_parent_record = parent_records.first or next 0
       partition_key = Dionysus::Producer::PartitionKey.new(example_parent_record, config: config)
         .to_key(responder: responder)
       key = Dionysus::Producer::Key.new(example_parent_record).to_key
@@ -101,9 +105,22 @@ class Dionysus::Producer::Outbox::Publisher
         responder.call(generate_message(parent_event_name, parent_record),
           options.merge(partition_key: partition_key, key: key))
       end
+      parent_records.size
     end
   end
 
+  # Deliberately does not raise: a legitimately deleted parent is indistinguishable from
+  # one that is not visible yet, and raising on the former would retry forever.
+  def handle_nothing_published(resource, event_name, topic)
+    message = "Published nothing for #{resource.class}, id: #{resource.id}, event: #{event_name}, " \
+              "topic: #{topic} - no responder matched and no parent could be resolved. " \
+              "Make sure it's processed later (e.g. by directly doing it from console)."
+    logger.error(message)
+    instrumenter.increment("dionysus.publish.nothing_published",
+      tags: ["resource:#{resource.class}", "topic:#{topic}"])
+    error_handler.capture_message(message)
+  end
+
   def publish_observer(observer_record, responder)
     publishable = Dionysus::Producer::Outbox::Publishable.new(observer_record)
     partition_key = Dionysus::Producer::PartitionKey.new(publishable).to_key(responder: responder)

data/lib/dionysus/producer/registry.rb

@@ -8,9 +8,9 @@ class Dionysus::Producer::Registry
     @container = {}
   end
 
-  def namespace(namespace, &block)
+  def namespace(namespace, &)
     registration = Registration.new(namespace)
-    registration.instance_eval(&block)
+    registration.instance_eval(&)
     container[namespace] = registration
   end
 
@@ -32,9 +32,9 @@ class Dionysus::Producer::Registry
       @serializer_klass = serializer_klass
     end
 
-    def topic(name, options = {}, &block)
+    def topic(name, options = {}, &)
       new_topic = Topic.new(namespace, name, serializer_klass, options)
-      new_topic.instance_eval(&block)
+      new_topic.instance_eval(&)
       producer = Dionysus::Producer::KarafkaResponderGenerator.new.generate(
         Dionysus::Producer.configuration, new_topic
       )
@@ -119,7 +119,7 @@ class Dionysus::Producer::Registry
             association_name = observer_config.fetch(:association_name)
             _attributes = observer_config.fetch(:attributes)
 
-            if association_name.is_a?(Symbol) && !model.instance_methods.include?(association_name.to_sym)
+            if association_name.is_a?(Symbol) && !model.method_defined?(association_name.to_sym)
               raise ArgumentError.new("association name :#{association_name} does not exist on model #{model}")
             end
           end
@@ -156,7 +156,7 @@ class Dionysus::Producer::Registry
             config_model_name = observer_config.fetch(:model).to_s
             config_attributes = observer_config.fetch(:attributes).to_a.map(&:to_sym)
 
-            config_model_name == resource_model_name && (config_attributes & changeset_attributes).any?
+            config_model_name == resource_model_name && config_attributes.intersect?(changeset_attributes)
           end
         end
       end

data/lib/dionysus/producer.rb

@@ -13,10 +13,10 @@ class Dionysus::Producer
     configuration.registry
   end
 
-  def self.declare(&config)
+  def self.declare(&)
     registry = Dionysus::Producer::Registry.new
 
-    registry.instance_eval(&config)
+    registry.instance_eval(&)
     configure do |configuration|
       configuration.registry = registry
     end

data/lib/dionysus/utils/null_instrumenter.rb

@@ -4,4 +4,6 @@ class Dionysus::Utils::NullInstrumenter
   def self.instrument(_name, _payload = {})
     yield
   end
+
+  def self.increment(_name, _options = {}); end
 end

data/lib/dionysus/version.rb

@@ -3,5 +3,5 @@
 module Dionysus
   module Version
   end
-  VERSION = "0.5.0"
+  VERSION = "1.1.0"
 end

data/lib/dionysus.rb

@@ -20,7 +20,7 @@ module Dionysus
     end
   end
 
-  def self.initialize_application!(environment:, seed_brokers:, client_id:, logger:, draw_routing: true, consumer_group_prefix: CONSUMER_GROUP_PREFIX)
+  def self.initialize_application!(environment:, seed_brokers:, client_id:, logger:, draw_routing: true, consumer_group_prefix: CONSUMER_GROUP_PREFIX, consumer_group_name: nil)
     ENV["KARAFKA_ENV"] = environment
 
     karafka_app = Class.new(Karafka::App) do
@@ -37,7 +37,10 @@ module Dionysus
 
     Object.const_set(:KarafkaApp, karafka_app)
     self.karafka_application = karafka_app
-    evaluate_routing(consumer_group_prefix: consumer_group_prefix) if consumer_registry.present? && draw_routing
+    return unless consumer_registry.present? && draw_routing
+
+    consumer_group_name ||= "#{consumer_group_prefix}_#{karafka_application.config.client_id}"
+    evaluate_routing(consumer_group_name: consumer_group_name)
   end
 
   def self.karafka_application=(karafka_app)
@@ -110,12 +113,11 @@ module Dionysus
     @monitor ||= Dionysus::Monitor.new
   end
 
-  def self.evaluate_routing(consumer_group_prefix:)
-    consumer_group_name = "#{consumer_group_prefix}_#{karafka_application.config.client_id}"
+  def self.evaluate_routing(consumer_group_name:)
     karafka_application.instance_exec(consumer_registry) do |registry|
       consumer_groups.draw do
         consumer_group consumer_group_name do
-          registry.registrations.each do |_, registration|
+          registry.registrations.each_value do |registration|
             registration.topics.each do |topic|
               topic topic.to_s do
                 consumer topic.consumer

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dionysus-rb
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-15 00:00:00.000000000 Z
+date: 2026-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ddtrace
+  name: datadog
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -366,14 +366,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -461,6 +461,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
+- ".ruby-version"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
@@ -571,14 +572,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: A framework on top of Karafka for Change Data Capture on the model level.