dionysus-rb 0.1.0

data/lib/dionysus/consumer/synced_data/assign_columns_from_synced_data.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::SyncedData::AssignColumnsFromSyncedData
+  attr_reader :config
+  private     :config
+
+  def initialize(config: Dionysus::Consumer.configuration)
+    @config = config
+  end
+
+  def call(collection, columns)
+    collection.each { |record| record.update!(hash_of_attributes(record, columns)) }
+  end
+
+  private
+
+  delegate :resolve_synced_data_hash_proc, to: :config
+
+  def hash_of_attributes(record, columns)
+    columns
+      .to_h { |column| [column, fetch_value_from_synced_data(record, column)] }
+  end
+
+  def fetch_value_from_synced_data(record, column)
+    resolve_synced_data_hash_proc.call(record).stringify_keys[column]
+  end
+end

data/lib/dionysus/consumer/synced_data/assign_columns_from_synced_data_job.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::SyncedData::AssignColumnsFromSyncedDataJob
+  include Sidekiq::Worker
+
+  sidekiq_options queue: Dionysus::Consumer::Config.default_sidekiq_queue
+
+  def self.enqueue(model_klass, columns, batch_size: 1000)
+    primary_key = model_klass.primary_key
+    model_klass.select(:id).find_in_batches(batch_size: batch_size).with_index do |records, index|
+      Dionysus.logger.info "[AssignColumnsFromSyncedDataJob] enqueue batch: #{index}"
+      model_name = model_klass.model_name.to_s
+      ids = records.map { |r| r.public_send(primary_key) }
+
+      set(queue: Dionysus::Consumer.configuration.sidekiq_queue)
+        .perform_async(model_name, ids, columns)
+    end
+  end
+
+  def perform(model_name, ids, columns)
+    model_klass = model_name.constantize
+    collection = model_klass.where(model_klass.primary_key => ids)
+
+    Dionysus::Consumer::SyncedData::AssignColumnsFromSyncedData.new.call(collection, columns)
+  end
+end

data/lib/dionysus/consumer/synced_data.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::SyncedData
+end

data/lib/dionysus/consumer/synchronizable_model.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::SynchronizableModel < SimpleDelegator
+  attr_reader :config
+  private :config
+
+  def initialize(config, model)
+    @config = config
+    super(model)
+  end
+
+  def model
+    __getobj__
+  end
+
+  def synced_at
+    if respond_to?(synced_updated_at_timestamp_attribute)
+      public_send(synced_updated_at_timestamp_attribute)
+    else
+      public_send(synced_created_at_timestamp_attribute)
+    end
+  end
+
+  def persist_with_dionysus?(event_updated_at)
+    (synced_at && event_updated_at && event_updated_at >= synced_at) || synced_at.nil? || event_updated_at.nil?
+  end
+
+  def assign_attributes_from_dionysus(attributes)
+    public_send("#{synced_data_attribute}=", attributes)
+    reverse_mapping = config.attributes_mapping_for_model(model.model_name).to_a.map(&:reverse).to_h
+
+    assignable_attributes = extract_assignable_attributes(attributes)
+      .map { |key, value| apply_mapping(reverse_mapping, key, value) }
+      .select { |attribute, _| respond_to?("#{attribute}=") }
+      .to_h
+
+    assign_attributes(assignable_attributes)
+  end
+
+  def remove_with_dionysus(deseralized_record)
+    if soft_deleteable_but_cannot_be_soft_deleted_via_attribute_assignment?(deseralized_record)
+      public_send(soft_delete_strategy)
+    elsif (deseralized_record.has_synced_canceled_at? && !respond_to?("#{soft_deleted_at_timestamp_attribute}=")) ||
+        !deseralized_record.has_synced_canceled_at?
+
+      destroy
+    end
+  end
+
+  def restore_with_dionysus
+    model.public_send("#{soft_deleted_at_timestamp_attribute}=", nil)
+  end
+
+  def restorable?(deseralized_record)
+    respond_to?("#{soft_deleted_at_timestamp_attribute}=") && !deseralized_record.has_synced_canceled_at?
+  end
+
+  private
+
+  delegate :soft_delete_strategy, :synced_created_at_timestamp_attribute,
+    :synced_updated_at_timestamp_attribute, :soft_deleted_at_timestamp_attribute,
+    :synced_data_attribute, to: :config
+
+  def soft_deleteable_but_cannot_be_soft_deleted_via_attribute_assignment?(deseralized_record)
+    (respond_to?(soft_delete_strategy) && !deseralized_record.has_synced_canceled_at?) ||
+      (respond_to?(soft_delete_strategy) && !respond_to?("#{soft_deleted_at_timestamp_attribute}="))
+  end
+
+  def extract_assignable_attributes(attributes)
+    attributes.clone.tap do |hash|
+      if synced_created_at_timestamp_attribute.to_s != "synced_created_at"
+        hash[synced_created_at_timestamp_attribute] =
+          hash["synced_created_at"]
+      end
+      if synced_updated_at_timestamp_attribute.to_s != "synced_updated_at"
+        hash[synced_updated_at_timestamp_attribute] =
+          hash["synced_updated_at"]
+      end
+      if soft_deleted_at_timestamp_attribute.to_s != "synced_canceled_at"
+        hash[soft_deleted_at_timestamp_attribute] =
+          hash["synced_canceled_at"]
+      end
+    end
+  end
+
+  def apply_mapping(reverse_mapping, key, value)
+    if reverse_mapping.key?(key.to_sym)
+      [reverse_mapping[key.to_sym], value]
+    else
+      [key, value]
+    end
+  end
+end

data/lib/dionysus/consumer/workers_group.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::WorkersGroup
+  attr_reader :workers
+  private :workers
+
+  def initialize
+    @workers = []
+  end
+
+  def <<(worker)
+    workers << worker
+  end
+
+  def work
+    workers.map(&:join)
+  end
+end

data/lib/dionysus/consumer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer
+  def self.configuration
+    @configuration ||= Dionysus::Consumer::Config.new
+  end
+
+  def self.configure
+    yield configuration
+  end
+
+  def self.registry
+    configuration.registry
+  end
+
+  def self.declare(&config)
+    registry = Dionysus::Consumer::Registry.new
+    registry.instance_eval(&config)
+
+    Dionysus.inject_routing!(registry)
+
+    configure do |configuration|
+      configuration.registry = registry
+    end
+  end
+
+  def self.reset!
+    return if registry.nil?
+
+    registry.registrations.values.flat_map(&:consumers).each do |consumer_class|
+      Dionysus.send(:remove_const, consumer_class.name.demodulize.to_sym) if consumer_class.name
+    end
+    @configuration = nil
+    Dionysus.inject_routing!(nil)
+  end
+end

data/lib/dionysus/monitor.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+class Dionysus::Monitor < Dry::Monitor::Notifications
+  EVENTS = %w[
+    outbox_producer.started
+    outbox_producer.stopped
+    outbox_producer.shutting_down
+    outbox_producer.error
+    outbox_producer.publishing_failed
+    outbox_producer.published
+    outbox_producer.processing_topic
+    outbox_producer.processed_topic
+    outbox_producer.lock_exists_for_topic
+    outbox_producer.heartbeat
+  ].freeze
+
+  private_constant :EVENTS
+
+  def initialize
+    super(:dionysus)
+    EVENTS.each { |event| register_event(event) }
+  end
+
+  def subscribe(event)
+    return super if events.include?(event.to_s)
+
+    raise UnknownEventError.new(events, event)
+  end
+
+  def events
+    EVENTS
+  end
+
+  class UnknownEventError < StandardError
+    attr_reader :available_events, :current_event
+    private     :available_events, :current_event
+
+    def initialize(available_events, current_event)
+      super()
+      @available_events = available_events
+      @current_event = current_event
+    end
+
+    def message
+      "unknown event: #{current_event}, the available events are: #{available_events.join(", ")}"
+    end
+  end
+end

data/lib/dionysus/producer/base_responder.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Based on the responder concept from Karafka 1.4
+class Dionysus::Producer::BaseResponder
+  class << self
+    attr_accessor :topics
+
+    def topic(topic_name)
+      self.topics ||= {}
+      self.topics[topic_name] = topic_name.to_s
+    end
+
+    def call(*data)
+      new.call(*data)
+    end
+  end
+
+  attr_reader :messages_buffer
+
+  def initialize
+    @messages_buffer = Hash.new { |h, k| h[k] = [] }
+  end
+
+  def call(*data)
+    respond(*data)
+    deliver
+  end
+
+  private
+
+  def deliver
+    messages_buffer.each_value do |data_elements|
+      data_elements.each do |data, options|
+        Karafka.producer.produce_sync(payload: data, **options)
+      end
+    end
+  end
+
+  def respond(*_data)
+    raise "implement me"
+  end
+
+  def respond_to(topic, data, options = {})
+    messages_buffer[topic] << [data.to_json, options.merge(topic: topic.to_s)]
+  end
+end

data/lib/dionysus/producer/config.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Config
+  attr_accessor :registry, :outbox_model, :database_connection_provider, :transaction_provider,
+    :datadog_statsd_client
+
+  attr_writer :instrumenter, :event_bus, :soft_delete_column, :default_partition_key, :outbox_worker_sleep_seconds,
+    :lock_client, :lock_expiry_time, :error_handler, :outbox_publishing_batch_size,
+    :transactional_outbox_enabled, :sidekiq_queue, :publisher_service_name,
+    :genesis_consistency_safety_delay, :hermes_event_producer, :publish_after_commit, :outbox_worker_publishing_delay,
+    :high_priority_sidekiq_queue, :observers_inline_maximum_size, :remove_consecutive_duplicates_before_publishing
+
+  def self.default_sidekiq_queue
+    :dionysus
+  end
+
+  def self.high_priority_sidekiq_queue
+    :dionysus_high_priority
+  end
+
+  def instrumenter
+    @instrumenter || Dionysus::Utils::NullInstrumenter
+  end
+
+  def event_bus
+    @event_bus || Dionysus::Utils::NullEventBus
+  end
+
+  def soft_delete_column
+    @soft_delete_column || "canceled_at"
+  end
+
+  def default_partition_key
+    @default_partition_key || :account_id
+  end
+
+  def outbox_worker_sleep_seconds
+    return BigDecimal("0.2") if @outbox_worker_sleep_seconds.nil?
+
+    @outbox_worker_sleep_seconds.to_d
+  end
+
+  def lock_client
+    @lock_client || Dionysus::Utils::NullLockClient
+  end
+
+  def lock_expiry_time
+    @lock_expiry_time || 10_000
+  end
+
+  def error_handler
+    @error_handler || Dionysus::Utils::NullErrorHandler
+  end
+
+  def outbox_publishing_batch_size
+    @outbox_publishing_batch_size || 100
+  end
+
+  def transactional_outbox_enabled
+    return @transactional_outbox_enabled if defined?(@transactional_outbox_enabled)
+
+    true
+  end
+
+  def sidekiq_queue
+    @sidekiq_queue || self.class.default_sidekiq_queue
+  end
+
+  def publisher_service_name
+    @publisher_service_name || Karafka.producer.id
+  end
+
+  def genesis_consistency_safety_delay
+    @genesis_consistency_safety_delay || 60.seconds
+  end
+
+  def hermes_event_producer
+    @hermes_event_producer || Dionysus::Utils::NullHermesEventProducer
+  end
+
+  def publish_after_commit
+    return @publish_after_commit if defined?(@publish_after_commit)
+
+    false
+  end
+
+  def outbox_worker_publishing_delay
+    (@outbox_worker_publishing_delay || 0).to_d.seconds
+  end
+
+  def high_priority_sidekiq_queue
+    @high_priority_sidekiq_queue || self.class.high_priority_sidekiq_queue
+  end
+
+  def observers_inline_maximum_size
+    @observers_inline_maximum_size || 1000
+  end
+
+  def remove_consecutive_duplicates_before_publishing
+    return @remove_consecutive_duplicates_before_publishing if defined?(@remove_consecutive_duplicates_before_publishing)
+
+    false
+  end
+end

data/lib/dionysus/producer/deleted_record_serializer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::DeletedRecordSerializer < Dionysus::Producer::ModelSerializer
+  def as_json
+    super.merge(primary_key => primary_key_value)
+  end
+
+  private
+
+  def primary_key
+    record.class.primary_key
+  end
+
+  def primary_key_value
+    record.public_send(primary_key)
+  end
+end

data/lib/dionysus/producer/genesis/performed.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+if defined?(Hermes)
+  class Dionysus::Producer::Genesis::Performed < Hermes::BaseEvent
+    attribute :model, Dry.Types::Strict::String
+    attribute :service, Dry.Types::Strict::String
+    attribute :topic, Dry.Types::Strict::String
+    attribute :start_at, Dry.Types::Nominal::DateTime
+    attribute :end_at, Dry.Types::Nominal::DateTime
+  end
+end

data/lib/dionysus/producer/genesis/stream_job.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Genesis::StreamJob
+  include Sidekiq::Worker
+
+  sidekiq_options queue: Dionysus::Producer::Config.default_sidekiq_queue
+
+  def perform(topic, model_klass, from, to, number_of_days, streamer_job)
+    Dionysus::Producer::Genesis::Streamer
+      .new(job_class: streamer_job.constantize)
+      .stream(topic, model_klass.constantize, from, to, number_of_days: number_of_days)
+  end
+end

data/lib/dionysus/producer/genesis/streamer/base_job.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Genesis::Streamer::BaseJob
+  include Sidekiq::Worker
+
+  sidekiq_options queue: Dionysus::Producer::Config.default_sidekiq_queue
+
+  ONE_DAY_IN_SECONDS = 60 * 60 * 24
+  BATCH_SIZE = 1000
+
+  def self.enqueue(relation, model_class, topic, number_of_days: 1, batch_size: BATCH_SIZE)
+    distributor = Dionysus::Utils::SidekiqBatchedJobDistributor.new(
+      batch_size: batch_size,
+      units_count: relation.count,
+      time_range_in_seconds: (ONE_DAY_IN_SECONDS * number_of_days)
+    )
+
+    relation.in_batches(of: batch_size).lazy.each_with_index do |batch_relation, batch_number|
+      distributor.enqueue_batch(
+        self,
+        Dionysus::Producer.configuration.sidekiq_queue,
+        batch_number,
+        batch_relation.pluck(model_class.primary_key).sort,
+        model_class.to_s,
+        topic
+      )
+    end
+  end
+
+  def perform(ids, resource_name, topic)
+    model_class = resource_name.constantize
+    primary_key_column = model_class.primary_key
+
+    model_class
+      .where(primary_key_column => ids)
+      .find_each { |entity| call(entity, topic) }
+  end
+
+  private
+
+  def call(_item, _topic)
+    raise "implement me!"
+  end
+end

data/lib/dionysus/producer/genesis/streamer/standard_job.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Genesis::Streamer::StandardJob < Dionysus::Producer::Genesis::Streamer::BaseJob
+  DESTROYED_EVENT_TYPE = "destroyed"
+  UPDATED_EVENT_TYPE = "updated"
+  GENESIS_TOPIC_SUFFIX = "genesis"
+
+  private
+
+  delegate :configuration, :outbox_model, :outbox_publisher, to: Dionysus::Producer
+  delegate :outbox_model, to: :configuration
+
+  def call(item, topic)
+    publishable = Dionysus::Producer::Outbox::Publishable.new(item)
+    outbox_record = Dionysus::Producer.configuration.outbox_model.new(
+      resource_class: publishable.model_name.to_s,
+      resource_id: publishable.publishable_id,
+      event_name: event_name_for(publishable),
+      topic: topic
+    )
+
+    options = {}
+    options[:genesis_only] = true if genesis_only?(topic)
+
+    outbox_publisher.publish(outbox_record, options)
+  end
+
+  def event_name_for(publishable)
+    Dionysus::Producer::Outbox::EventName
+      .new(publishable.resource_name)
+      .for_event_type(event_type(publishable))
+  end
+
+  def event_type(publishable)
+    return DESTROYED_EVENT_TYPE if publishable.soft_deletable? && publishable.soft_deleted?
+
+    UPDATED_EVENT_TYPE
+  end
+
+  def genesis_only?(topic)
+    topic.ends_with?(GENESIS_TOPIC_SUFFIX)
+  end
+end

data/lib/dionysus/producer/genesis/streamer.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Genesis::Streamer
+  attr_reader :job_class, :config
+  private     :job_class, :config
+
+  def initialize(job_class: Dionysus::Producer::Genesis::Streamer::StandardJob,
+    config: Dionysus::Producer.configuration)
+    @job_class = job_class
+    @config = config
+  end
+
+  def stream(topic, model_class, from, to, options = {})
+    resources = fetch_resources(model_class, from, to, options)
+    job_class.enqueue(
+      resources,
+      model_class,
+      topic,
+      number_of_days: options.fetch(:number_of_days, 1),
+      batch_size: options.fetch(:batch_size, 1000)
+    )
+  end
+
+  private
+
+  delegate :soft_delete_column, to: :config
+
+  def fetch_resources(resource_class, from, to, options_hash)
+    records = resource_class
+    records = resource_class.where("updated_at BETWEEN ? AND ?", from, to) if from.present? && to.present?
+    if visible_only?(options_hash) && records.column_names.include?(soft_delete_column.to_s)
+      records = records.where(soft_delete_column => nil)
+    end
+    records
+  end
+
+  def visible_only?(options_hash)
+    options_hash.fetch(:visible_only, false)
+  end
+end

data/lib/dionysus/producer/genesis.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Genesis
+  attr_reader :config
+  private :config
+
+  def initialize(config: Dionysus::Producer.configuration)
+    @config = config
+  end
+
+  def stream(topic:, model:, number_of_days:, from: nil, to: nil,
+    streamer_job: Dionysus::Producer::Genesis::Streamer::StandardJob)
+    ensure_genesis_not_executed_for_model_that_is_only_a_dependency(model, topic)
+    enqueue_stream_model_for_topic(topic, model, from, to, number_of_days, streamer_job)
+    publish_genesis_performed(model: model, topic: topic, number_of_days: number_of_days)
+  end
+
+  private
+
+  delegate :publisher_service_name, :genesis_consistency_safety_delay, :hermes_event_producer, :sidekiq_queue,
+    to: :config
+  delegate :responders_for_model_for_topic, :responders_for_dependency_parent_for_topic,
+    to: Dionysus::Producer
+
+  def ensure_genesis_not_executed_for_model_that_is_only_a_dependency(model, topic)
+    if responders_for_model_for_topic(model,
+      topic).empty? && responders_for_dependency_parent_for_topic(model, topic).any?
+      raise CannotExecuteGenesisForModelThatIsOnlyDependency.new(model, topic)
+    end
+  end
+
+  def enqueue_stream_model_for_topic(topic, model, from, to, number_of_days, streamer_job)
+    Dionysus::Producer::Genesis::StreamJob
+      .set(queue: sidekiq_queue)
+      .perform_async(topic.to_s, model.to_s, from.as_json, to.as_json, number_of_days.to_i, streamer_job.to_s)
+  end
+
+  def publish_genesis_performed(model:, topic:, number_of_days:)
+    event = Dionysus::Producer::Genesis::Performed.new(
+      model: model.to_s,
+      service: publisher_service_name,
+      topic: topic,
+      start_at: Time.current,
+      end_at: Time.current + number_of_days.days + genesis_consistency_safety_delay
+    )
+    hermes_event_producer.publish(event)
+  end
+
+  class CannotExecuteGenesisForModelThatIsOnlyDependency < StandardError
+    attr_reader :model, :topic
+    private     :model, :topic
+    def initialize(model, topic)
+      super()
+      @model = model
+      @topic = topic
+    end
+
+    def message
+      "Cannot execute genesis for model #{model}, #{topic} because that is only a dependency. Execute it for parent instead"
+    end
+  end
+end