dionysus-rb 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf3312592aa3f39392dc5ea53bb55a026cdf1dbde5cce0fc598f50a99d1d499e
-  data.tar.gz: 0c302683ed17704b861bd1d08eb3d87f549e7102af46acd23cccd1be0163c329
+  metadata.gz: '0934079a452c75d517f1954e9cc6650c4b6219273c329013b676f336ea0d9e13'
+  data.tar.gz: bee752d4720e8f4249867ecf3506b725fa9ba9cd678ebadf9ac0ac893c635925
 SHA512:
-  metadata.gz: d788ee2c0aadac5bd2d8510517f941043ef032cb22d049ed8457a5ce68a26cd799ef4370fd58cb89f2999ade8e4dfd174fcebe39d42ed702f4e235060dc877f1
-  data.tar.gz: a3e25434d96c3579b6185b9e6cfd6d6bd4e87aba52eb7e5e8b7736addf25aada4964bbf1ada59d9c4ccf0051b5b251bfe207669ce08d4852836ca35e8e5dc1d9
+  metadata.gz: e12c164f5909f35e9b0d4336fb31ebb3d79e064409243beaad6e40c32f5590ff296ee805601de312b111fcc5fbb10d86008064cb2c229de56d7eab8518e6c9f9
+  data.tar.gz: 6a0fffb5d3b4d65eab34a0aaa16871002b8713addb45e07a1094706559fc978d5b37c885d6d2179c481bea436eddfe9805db0385a464ddb6a70b056ccf10e489

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## [Unreleased]
 
+## [0.4.0]
+- Allow `Dionysus::Producer::Genesis::StreamJob`/`Dionysus::Producer::Genesis::Streamer` to take more options and perform filtering by extra conditions. This is useful if you only need to stream some of the records.
+
+## [0.3.0]
+- Allow to provide multiple message filters. `message_filter` stays for backwards compatibility and depends on `message_filters`.
+
 ## [0.2.0]
 - Fix creation of outbox records when updating a record that was soft-deleted
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dionysus-rb (0.2.0)
+    dionysus-rb (0.4.0)
       activerecord (>= 5)
       activesupport (>= 3.2)
       concurrent-ruby

data/README.md

@@ -1136,7 +1136,8 @@ Dionysus::Consumer.configure do |config|
   config.synced_data_attribute = :synced_data # required, default: :synced_data
   config.resolve_synced_data_hash_proc = ->(record) { record.synced_data_model.synced_data_hash } # optional, defaults to ->(record) { record.public_send(Dionysus::Consumer.configuration.synced_data_attribute).to_h }
   config.sidekiq_queue = :default # optional, defaults to `:dionysus`
-  config.message_filter = FilterIgnoringLargeMessageToAvoidOutofMemoryErrors.new(error_handler: Sentry) # not required, defaults to Dionysus::Utils::DefaultMessageFilter, which doesn't ignore any messages. It can be useful when you want to ignore some messages, e.g. some very large ones that would cause OOM error. Check the implementation of `Dionysus::Utils::DefaultMessageFilter for more details to understand what kind of arguments are available to set the condition. `error_handler` needs to implement Sentry-like interface.
+  config.message_filter = FilterIgnoringLargeMessageToAvoidOutofMemoryErrors.new(error_handler: Sentry) #  DEPRECATED - not required, defaults to Dionysus::Utils::DefaultMessageFilter, which doesn't ignore any messages. It can be useful when you want to ignore some messages, e.g. some very large ones that would cause OOM error. Check the implementation of `Dionysus::Utils::DefaultMessageFilter for more details to understand what kind of arguments are available to set the condition. `error_handler` needs to implement Sentry-like interface. Kept for backwards compatibility, please use `message_filters` instead.
+  config.message_filters = [FilterIgnoringLargeMessageToAvoidOutofMemoryErrors.new(error_handler: Sentry)] # not required, defaults to [Dionysus::Utils::DefaultMessageFilter], which doesn't ignore any messages. It can be useful when you want to ignore some messages, e.g. some very large ones that would cause OOM error. Check the implementation of `Dionysus::Utils::DefaultMessageFilter for more details to understand what kind of arguments are available to set the condition. `error_handler` needs to implement Sentry-like interface. 
 
   # if you ever need to provide mapping:
 

data/lib/dionysus/consumer/config.rb

@@ -6,7 +6,7 @@ class Dionysus::Consumer::Config
     :instrumenter, :event_bus, :soft_delete_strategy, :soft_deleted_at_timestamp_attribute,
     :synced_created_at_timestamp_attribute, :synced_updated_at_timestamp_attribute, :synced_id_attribute,
     :synced_data_attribute, :consumer_base_class, :retry_provider, :resolve_synced_data_hash_proc, :sidekiq_queue,
-    :message_filter
+    :message_filters
 
   def self.default_sidekiq_queue
     :dionysus
@@ -94,4 +94,13 @@ class Dionysus::Consumer::Config
     @message_filter || Dionysus::Utils::DefaultMessageFilter.new(error_handler:
       Dionysus::Utils::NullErrorHandler)
   end
+
+  def message_filter=(val)
+    @message_filter = val
+    self.message_filters = [val]
+  end
+
+  def message_filters
+    @message_filters || Array.wrap(message_filter)
+  end
 end

data/lib/dionysus/consumer/params_batch_processor.rb

@@ -27,8 +27,9 @@ class Dionysus::Consumer::ParamsBatchProcessor
             transformed_data = Dionysus::Consumer::Deserializer.new(data).deserialize
           end
 
-          if ignore_message?(topic: topic, message: message, transformed_data: transformed_data)
-            notify_about_ignored_message(topic: topic, message: message, transformed_data: transformed_data)
+          if (applicable_message_filter = find_applicable_message_filter(topic, message, transformed_data))
+            applicable_message_filter.notify_about_ignored_message(topic: topic, message: message,
+              transformed_data: transformed_data)
             next
           end
 
@@ -50,8 +51,11 @@ class Dionysus::Consumer::ParamsBatchProcessor
 
   private
 
-  delegate :message_filter, to: :config
-  delegate :ignore_message?, :notify_about_ignored_message, to: :message_filter
+  delegate :message_filters, to: :config
+
+  def find_applicable_message_filter(topic, message, transformed_data)
+    message_filters.find { |f| f.ignore_message?(topic: topic, message: message, transformed_data: transformed_data) }
+  end
 
   def instrument(label, options = {}, &block)
     config.instrumenter.instrument(label, options, &block)

data/lib/dionysus/producer/genesis/stream_job.rb

@@ -5,9 +5,10 @@ class Dionysus::Producer::Genesis::StreamJob
 
   sidekiq_options queue: Dionysus::Producer::Config.default_sidekiq_queue
 
-  def perform(topic, model_klass, from, to, number_of_days, streamer_job)
+  def perform(topic, model_klass, from, to, number_of_days, streamer_job, options = {})
+    final_options = options.symbolize_keys.reverse_merge(number_of_days: number_of_days)
     Dionysus::Producer::Genesis::Streamer
       .new(job_class: streamer_job.constantize)
-      .stream(topic, model_klass.constantize, from, to, number_of_days: number_of_days)
+      .stream(topic, model_klass.constantize, from, to, final_options)
   end
 end

data/lib/dionysus/producer/genesis/streamer.rb

@@ -26,14 +26,31 @@ class Dionysus::Producer::Genesis::Streamer
   delegate :soft_delete_column, to: :config
 
   def fetch_resources(resource_class, from, to, options_hash)
-    records = resource_class
-    records = resource_class.where("updated_at BETWEEN ? AND ?", from, to) if from.present? && to.present?
+    resource_class
+      .then { |records| apply_time_range(records, from, to) }
+      .then { |records| apply_visibility(records, options_hash) }
+      .then { |records| apply_query_conditions(records, options_hash) }
+  end
+
+  def apply_time_range(records, from, to)
+    records = records.where("updated_at BETWEEN ? AND ?", from, to) if from.present? && to.present?
+    records
+  end
+
+  def apply_visibility(records, options_hash)
     if visible_only?(options_hash) && records.column_names.include?(soft_delete_column.to_s)
       records = records.where(soft_delete_column => nil)
     end
     records
   end
 
+  def apply_query_conditions(records, options_hash)
+    if (query_conditions = options_hash.fetch(:query_conditions, {})).any?
+      query_conditions.each { |attr, val| records = records.where(attr => val) }
+    end
+    records
+  end
+
   def visible_only?(options_hash)
     options_hash.fetch(:visible_only, false)
   end

data/lib/dionysus/producer/genesis.rb

@@ -32,7 +32,7 @@ class Dionysus::Producer::Genesis
   def enqueue_stream_model_for_topic(topic, model, from, to, number_of_days, streamer_job)
     Dionysus::Producer::Genesis::StreamJob
       .set(queue: sidekiq_queue)
-      .perform_async(topic.to_s, model.to_s, from.as_json, to.as_json, number_of_days.to_i, streamer_job.to_s)
+      .perform_async(topic.to_s, model.to_s, from.as_json, to.as_json, number_of_days.to_i, streamer_job.to_s, {})
   end
 
   def publish_genesis_performed(model:, topic:, number_of_days:)

data/lib/dionysus/version.rb

@@ -3,5 +3,5 @@
 module Dionysus
   module Version
   end
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dionysus-rb
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-21 00:00:00.000000000 Z
+date: 2024-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord