dionysus-rb 0.1.0

data/lib/dionysus/consumer/deserializer.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/array/wrap"
+
+class Dionysus::Consumer::Deserializer
+  attr_reader :data
+
+  def initialize(data)
+    @data = data.to_a
+  end
+
+  def deserialize
+    data.map { |serialized_payload| SerializedRecord.new(serialized_payload) }.map do |serialized_record|
+      transformed_payload = DeserializedRecord.new
+      transformed_payload = populate_attributes_with_relationships(serialized_record, transformed_payload)
+
+      transformed_payload = assign_attributes(serialized_record, transformed_payload)
+
+      serialized_record.expected_has_many_relationships.each do |relationship_name|
+        deserialize_has_many_relationship(serialized_record, relationship_name, transformed_payload)
+      end
+
+      serialized_record.expected_has_one_relationships.each do |relationship_name|
+        deserialize_has_one_relationship(serialized_record, relationship_name, transformed_payload)
+      end
+
+      transformed_payload
+    end
+  end
+
+  private
+
+  def populate_attributes_with_relationships(serialized_record, transformed_payload)
+    serialized_record.links.each do |relationship_name, value|
+      transformed_payload.populate_attributes_with_relationship(relationship_name, value)
+    end
+
+    transformed_payload
+  end
+
+  def assign_attributes(serialized_record, transformed_payload)
+    transformed_payload.synced_id = serialized_record.id
+    transformed_payload.synced_created_at = serialized_record.created_at if serialized_record.has_created_at?
+    transformed_payload.synced_updated_at = serialized_record.updated_at if serialized_record.has_updated_at?
+    transformed_payload.synced_canceled_at = serialized_record.canceled_at if serialized_record.has_canceled_at?
+
+    serialized_record.plain_attributes.each do |attribute, value|
+      transformed_payload.attributes[attribute] = value
+    end
+
+    transformed_payload
+  end
+
+  def deserialize_has_many_relationship(serialized_record, relationship_name, transformed_payload)
+    value = serialized_record[relationship_name]
+    deserialized_relationship = (value && Dionysus::Consumer::Deserializer.new(value).deserialize) || nil
+    relationship_model_name = serialized_record.model_name_for_relationship(relationship_name)
+    transformed_payload.has_many << [relationship_model_name, deserialized_relationship]
+    transformed_payload.delete(relationship_name)
+    transformed_payload
+  end
+
+  def deserialize_has_one_relationship(serialized_record, relationship_name, transformed_payload)
+    value = serialized_record[relationship_name]
+    deserialized_relationship = Dionysus::Consumer::Deserializer.new(Array.wrap(value)).deserialize
+    relationship_model_name = serialized_record.model_name_for_relationship(relationship_name)
+    transformed_payload.has_one << [relationship_model_name, deserialized_relationship.first]
+    transformed_payload.delete(relationship_name)
+    transformed_payload
+  end
+
+  class DeserializedRecord < SimpleDelegator
+    def initialize
+      super(canonical_format)
+    end
+
+    def transformed_payload
+      __getobj__
+    end
+
+    def attributes
+      transformed_payload.fetch(:attributes)
+    end
+
+    def has_many
+      transformed_payload.fetch(:has_many)
+    end
+
+    def has_one
+      transformed_payload.fetch(:has_one)
+    end
+
+    def synced_id
+      attributes.fetch("synced_id") { "synced_id not found in #{attributes}! Something is very wrong." }
+    end
+
+    def synced_id=(val)
+      attributes["synced_id"] = val
+    end
+
+    def synced_created_at
+      attributes["synced_created_at"]
+    end
+
+    def synced_created_at=(val)
+      attributes["synced_created_at"] = val
+    end
+
+    def synced_updated_at
+      attributes["synced_updated_at"]
+    end
+
+    def synced_updated_at=(val)
+      attributes["synced_updated_at"] = val
+    end
+
+    def synced_canceled_at
+      attributes["synced_canceled_at"]
+    end
+
+    def synced_canceled_at=(val)
+      attributes["synced_canceled_at"] = val
+    end
+
+    def populate_attributes_with_relationship(relationship_name, value)
+      if value.respond_to?(:to_hash)
+        attributes["synced_#{relationship_name}_id"] = value["id"]
+        attributes["synced_#{relationship_name}_type"] = value["type"]
+      elsif value.respond_to?(:to_ary)
+        relationship_name = ActiveSupport::Inflector.singularize(relationship_name)
+        attributes["synced_#{relationship_name}_ids"] = value
+      else
+        attributes["synced_#{relationship_name}_id"] = value
+      end
+    end
+
+    def has_synced_canceled_at?
+      attributes.key?("synced_canceled_at")
+    end
+
+    private
+
+    def canonical_format
+      { attributes: {}, has_many: [], has_one: [] }
+    end
+  end
+
+  class SerializedRecord
+    RESERVED_ATTRIBUTES = %w[links id created_at updated_at canceled_at].freeze
+    private_constant :RESERVED_ATTRIBUTES
+
+    attr_reader :payload
+    private     :payload
+
+    delegate :key?, :[], to: :payload
+
+    def initialize(payload)
+      @payload = payload
+    end
+
+    def plain_attributes
+      payload.except(*RESERVED_ATTRIBUTES, *expected_has_many_relationships, *expected_has_one_relationships)
+    end
+
+    def expected_has_many_relationships
+      to_many_foreign_keys.keys
+    end
+
+    def expected_has_one_relationships
+      to_one_foreign_keys.keys
+    end
+
+    def model_name_for_relationship(relationship_name)
+      if polymorphic_relationship?(relationship_name) && links[relationship_name].key?("type")
+        links[relationship_name]["type"]
+      else
+        relationship_name
+      end
+    end
+
+    def id
+      payload["id"]
+    end
+
+    def created_at
+      payload["created_at"]
+    end
+
+    def updated_at
+      payload["updated_at"]
+    end
+
+    def canceled_at
+      payload["canceled_at"]
+    end
+
+    def has?(key)
+      key?(key)
+    end
+
+    def has_created_at?
+      has?("created_at")
+    end
+
+    def has_updated_at?
+      has?("updated_at")
+    end
+
+    def has_canceled_at?
+      has?("canceled_at")
+    end
+
+    def links
+      payload["links"].to_h
+    end
+
+    private
+
+    def to_one_foreign_keys
+      links.reject { |_, value| value.respond_to?(:to_ary) }
+    end
+
+    def to_many_foreign_keys
+      links.select { |_, value| value.respond_to?(:to_ary) }
+    end
+
+    def polymorphic_relationship?(relationship_name)
+      links[relationship_name]&.respond_to?(:to_hash)
+    end
+  end
+end

data/lib/dionysus/consumer/dionysus_event.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::DionysusEvent
+  attr_reader :event_name, :model_name, :transformed_data, :local_changes
+
+  def initialize(event_name, model_name, transformed_data, aggregate_root: true)
+    @event_name = event_name.to_s
+    @model_name = model_name.to_s
+    @transformed_data = transformed_data
+    @local_changes = {}
+    @aggregate_root = aggregate_root
+  end
+
+  def created?
+    event_name.end_with?("created")
+  end
+
+  def updated?
+    event_name.end_with?("updated")
+  end
+
+  def destroyed?
+    event_name.end_with?("destroyed")
+  end
+
+  def generic_event?
+    created? || updated? || destroyed?
+  end
+
+  def aggregate_root?
+    @aggregate_root == true
+  end
+
+  def to_h
+    {
+      event_name: event_name,
+      model_name: model_name,
+      transformed_data: transformed_data,
+      local_changes: local_changes
+    }
+  end
+end

data/lib/dionysus/consumer/karafka_consumer_generator.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::KarafkaConsumerGenerator
+  TOMBSTONE = nil
+
+  def generate(config, topic)
+    base_class = topic.consumer_base_class || config.consumer_base_class
+
+    consumer_klass = Class.new(base_class) do
+      define_method :consume do
+        config.retry_provider.retry do
+          processed_events = Concurrent::Array.new
+          config.instrumenter.instrument("dionysus.consume.#{topic}") do
+            batch_number = 0
+
+            if topic.concurrency
+              workers = Dionysus::Consumer::WorkersGroup.new
+              messages.each do |batch|
+                batch_number += 1 # cannot use each_with_index on params_batch
+                worker = Thread.new do
+                  Thread.current.report_on_exception = true
+                  Thread.current.abort_on_exception = true
+                  processed_events.concat(process_batch(config, topic, batch, batch_number))
+                end
+                workers << worker
+              end
+              workers.work
+            else
+              final_params_batch = topic.params_batch_transformation&.call(messages) || messages
+              final_params_batch.each do |batch|
+                batch_number += 1 # cannot use each_with_index on params_batch
+                processed_events.concat(process_batch(config, topic, batch, batch_number))
+              end
+            end
+          end
+          Dionysus::Consumer::BatchEventsPublisher.new(config, topic).publish(processed_events)
+        end
+      end
+
+      private
+
+      define_method :process_batch do |configuration, current_topic, batch, batch_number|
+        configuration.transaction_provider.transaction do
+          Dionysus::Consumer::ParamsBatchProcessor.new(configuration, current_topic).process(batch,
+            batch_number)
+        end
+      end
+    end
+
+    consumer_klass_name = "#{topic.to_s.classify}Consumer"
+
+    Dionysus.send(:remove_const, consumer_klass_name) if Dionysus.const_defined?(consumer_klass_name)
+    Dionysus.const_set(consumer_klass_name, consumer_klass)
+    consumer_klass
+  end
+end

data/lib/dionysus/consumer/params_batch_processor.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::ParamsBatchProcessor
+  attr_reader :config, :topic
+  private     :config, :topic
+
+  def initialize(config, topic)
+    @config = config
+    @topic = topic
+  end
+
+  def process(batch, batch_number)
+    processed_events = []
+    instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}") do
+      payload = batch.payload.to_h
+      metadata = batch.metadata
+      message = payload["message"].to_a
+
+      with_mutex(metadata, config) do
+        message.each do |current_event|
+          event_name = current_event["event"]
+          data = Array.wrap(current_event["data"])
+          model_name = current_event["model_name"]
+
+          transformed_data = nil
+          config.instrumenter.instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.deserialize") do
+            transformed_data = Dionysus::Consumer::Deserializer.new(data).deserialize
+          end
+
+          if ignore_message?(topic: topic, message: message, transformed_data: transformed_data)
+            notify_about_ignored_message(topic: topic, message: message, transformed_data: transformed_data)
+            next
+          end
+
+          dionysus_event = Dionysus::Consumer::DionysusEvent.new(event_name, model_name,
+            transformed_data)
+          config.instrumenter.instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist",
+            dionysus_event.to_h.except(:transformed_data, :local_changes)) do
+            config.transaction_provider.connection_pool.with_connection do
+              Dionysus::Consumer::Persistor.new(config, topic).persist(dionysus_event,
+                batch_number)
+            end
+          end
+          processed_events << dionysus_event
+        end
+      end
+    end
+    processed_events
+  end
+
+  private
+
+  delegate :message_filter, to: :config
+  delegate :ignore_message?, :notify_about_ignored_message, to: :message_filter
+
+  def instrument(label, options = {}, &block)
+    config.instrumenter.instrument(label, options, &block)
+  end
+
+  def with_mutex(metadata, config, &block)
+    message_key = metadata.key || SecureRandom.uuid
+    config.processing_mutex_provider.send(config.processing_mutex_method_name, "Dionysus-#{message_key}",
+      &block)
+  end
+end

data/lib/dionysus/consumer/params_batch_transformations/remove_duplicates_strategy.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy
+  def call(params_batch)
+    return params_batch if duplicates_removal_not_applicable?(params_batch)
+
+    Karafka::Messages::Messages.new(transform_messages_array(params_batch), params_batch.metadata)
+  end
+
+  private
+
+  # the idea is following:
+  # 1. group messages by event and id - for a given model we can expect unique messages for _created and _deleted ecents
+  #   but we can have multiple _updated events, so this is where we are interested in removing duplicates
+  # 2. sort each group by updated_at, reverse (because the sort is ascending) and take the first one -
+  #   this way we will have the most recent update
+  # 3. flatten the array of arrays as all groups will have a single item
+  # It is safer to apply sorting just for the _updated events to a given model as otherwise we could change the order of
+  #   the messages for a different type which might be not desirable
+  def transform_messages_array(params_batch)
+    params_batch
+      .to_a
+      .group_by { |batch| grouping_key_by_event_and_id(batch) }
+      .map { |_, group| group.max_by { |batch| updated_at_from_batch(batch) } }
+      .flatten
+  end
+
+  def grouping_key_by_event_and_id(batch)
+    [
+      batch.payload.fetch("message").first.fetch("event"),
+      batch.payload.fetch("message").first["data"].first.fetch("id", nil)
+    ].join
+  end
+
+  def updated_at_from_batch(batch)
+    timestamp = batch.payload.fetch("message", []).first.to_h.fetch("data", []).first.to_h.fetch("updated_at", nil)
+
+    return timestamp.to_datetime if timestamp.respond_to?(:to_datetime)
+
+    Time.current
+  end
+
+  def duplicates_removal_not_applicable?(params_batch)
+    any_message_containing_more_than_one_event?(params_batch) || any_event_containing_more_than_one_item(params_batch)
+  end
+
+  def any_message_containing_more_than_one_event?(params_batch)
+    params_batch.any? { |batch| batch.payload.to_h.fetch("message", []).size != 1 }
+  end
+
+  def any_event_containing_more_than_one_item(params_batch)
+    params_batch.any? { |batch| batch.payload.to_h.fetch("message", []).first.to_h.fetch("data", []).size != 1 }
+  end
+end

data/lib/dionysus/consumer/params_batch_transformations.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::ParamsBatchTransformations
+end

data/lib/dionysus/consumer/persistor.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::Persistor
+  attr_reader :config, :topic
+  private     :config, :topic
+
+  def initialize(config, topic)
+    @config = config
+    @topic = topic
+  end
+
+  def persist(dionysus_event, batch_number)
+    if dionysus_event.generic_event?
+      if dionysus_event.created? && topic.options[:import] == true
+        persist_via_dionysus_create(dionysus_event, batch_number)
+      elsif dionysus_event.destroyed? && topic.options[:import] == true
+        persist_via_dionysus_destroy(dionysus_event, batch_number)
+      else
+        persist_standard_event(dionysus_event, batch_number)
+      end
+    else
+      log_unknown_event_type(dionysus_event)
+    end
+  end
+
+  private
+
+  def persist_via_dionysus_create(dionysus_event, batch_number)
+    model_klass = find_model_klass(dionysus_event) or return
+    config.instrumenter.instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.dionysus_import") do
+      model_klass.dionysus_import(dionysus_event.transformed_data)
+    end
+  end
+
+  def persist_via_dionysus_destroy(dionysus_event, batch_number)
+    model_klass = find_model_klass(dionysus_event) or return
+    config.instrumenter.instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.dionysus_destroy") do
+      model_klass.dionysus_destroy(dionysus_event.transformed_data)
+    end
+  end
+
+  def find_model_klass(dionysus_event)
+    config.model_factory.for_model(dionysus_event.model_name)
+  end
+
+  def persist_standard_event(dionysus_event, batch_number)
+    Array.wrap(dionysus_event.transformed_data).each do |deseralized_record|
+      model_klass = find_model_klass(dionysus_event) or return
+      attributes = deseralized_record.attributes
+      has_one_relationships = deseralized_record.has_one
+      has_many_relationships = deseralized_record.has_many
+      synced_id = deseralized_record.synced_id
+
+      if synced_id.nil?
+        Dionysus.logger.error("[Dionysus] synced_id nil for #{deseralized_record}, that should never happen!")
+        next
+      end
+
+      record = Dionysus::Consumer::SynchronizableModel.new(config,
+        model_klass.find_or_initialize_by(config.synced_id_attribute => synced_id))
+      event_updated_at = deseralized_record.synced_updated_at || deseralized_record.synced_created_at
+
+      next unless record.persist_with_dionysus?(event_updated_at)
+
+      record.assign_attributes_from_dionysus(attributes)
+      if dionysus_event.destroyed?
+        instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.remove_with_dionysus") do
+          record.remove_with_dionysus(deseralized_record) if dionysus_event.aggregate_root?
+        end
+      else
+        instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.restore_with_dionysus") do
+          record.restore_with_dionysus if record.restorable?(deseralized_record)
+        end
+      end
+
+      dionysus_event.local_changes[[dionysus_event.model_name, synced_id]] = record.changes if record.changes.present?
+
+      instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.save") do
+        record.save unless record.destroyed?
+      end
+
+      instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.persist_to_many_relationships") do
+        has_many_relationships.each do |relationship_name, relationship_records|
+          persist_to_many_relationship(dionysus_event, relationship_name, record, relationship_records,
+            batch_number)
+        end
+      end
+
+      instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.persist_to_one_relationships") do
+        has_one_relationships.each do |relationship_name, relationship_record|
+          persist_to_one_relationship(dionysus_event, relationship_name, record, relationship_record,
+            batch_number)
+        end
+      end
+    end
+  end
+
+  def log_unknown_event_type(dionysus_event)
+    Dionysus.logger.debug("[Dionysus] unknown event type #{dionysus_event.event_name}")
+  end
+
+  def persist_to_one_relationship(original_event, relationship_name, parent_model_record, record, batch_number)
+    instrumentation_arguments = {
+      event_name: original_event.event_name,
+      parent_model_record: parent_model_record.model_name.to_s,
+      relationship_name: relationship_name
+    }
+    instrument(
+      "dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.persist_to_one_relationship.#{relationship_name}", instrumentation_arguments
+    ) do
+      return if parent_model_record.nil? || record.nil?
+
+      records = Array.wrap(record)
+      dionysus_event = Dionysus::Consumer::DionysusEvent.new(original_event.event_name,
+        relationship_name, records, aggregate_root: false)
+
+      instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.persist_to_one_relationship.#{relationship_name}.persist") do
+        persist(dionysus_event, batch_number)
+        original_event.local_changes.merge!(dionysus_event.local_changes)
+      end
+
+      instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.persist_to_one_relationship.#{relationship_name}.resolve_to_one_association") do
+        parent_model_record.resolve_to_one_association(relationship_name, record.synced_id)
+      end
+    end
+  end
+
+  def persist_to_many_relationship(original_event, relationship_name, parent_model_record, records, batch_number)
+    instrumentation_arguments = {
+      event_name: original_event.event_name,
+      parent_model_record: parent_model_record.model_name.to_s,
+      relationship_name: relationship_name
+    }
+    instrument(
+      "dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.persist_to_many_relationship.#{relationship_name}", instrumentation_arguments
+    ) do
+      return if parent_model_record.nil? || records.nil?
+
+      dionysus_event = Dionysus::Consumer::DionysusEvent.new(original_event.event_name,
+        relationship_name, records, aggregate_root: false)
+
+      instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.persist_to_many_relationship.#{relationship_name}.persist") do
+        persist(dionysus_event, batch_number)
+        original_event.local_changes.merge!(dionysus_event.local_changes)
+      end
+
+      synced_ids_of_related_records = records.map(&:synced_id)
+      instrument("dionysus.consume.#{topic}.batch_number_#{batch_number}.persist.persist_to_many_relationship.#{relationship_name}.resolve_to_many_association") do
+        parent_model_record.resolve_to_many_association(relationship_name, synced_ids_of_related_records)
+      end
+    end
+  end
+
+  def instrument(label, options = {}, &block)
+    config.instrumenter.instrument(label, options, &block)
+  end
+end

data/lib/dionysus/consumer/registry.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+class Dionysus::Consumer::Registry
+  attr_reader :container
+  private :container
+
+  def initialize
+    @container = {}
+  end
+
+  def namespace(namespace, &block)
+    registration = Registration.new(namespace)
+    registration.instance_eval(&block)
+    container[namespace] = registration
+  end
+
+  def registrations
+    container
+  end
+
+  class Registration
+    attr_reader :namespace, :topics, :deserializer_klass, :consumers
+
+    def initialize(namespace)
+      @namespace = namespace
+      @topics = []
+      @deserializer_klass = nil
+      @consumers = []
+    end
+
+    def deserializer(deserializer_klass)
+      @deserializer_klass = deserializer_klass
+    end
+
+    def topic(name, options = {}, &block)
+      new_topic = Topic.new(namespace, name, deserializer_klass, options, &block)
+      consumer = Dionysus::Consumer::KarafkaConsumerGenerator.new.generate(
+        Dionysus::Consumer.configuration, new_topic
+      )
+      consumers << consumer
+      new_topic.consumer = consumer
+      topics << new_topic
+    end
+
+    class Topic
+      attr_reader :namespace, :name, :deserializer_klass, :options, :extensions_block
+
+      attr_accessor :consumer
+
+      def initialize(namespace, name, deserializer_klass, options = {}, &block)
+        @namespace = namespace
+        @name = name
+        @deserializer_klass = deserializer_klass
+        @options = options
+        @extensions_block = block
+      end
+
+      def to_s
+        Dionysus::TopicName.new(namespace, name).to_s
+      end
+
+      def sidekiq_worker
+        options.fetch(:worker, nil)
+      end
+
+      def sidekiq_backend?
+        options.fetch(:sidekiq, false)
+      end
+
+      def consumer_base_class
+        options.fetch(:consumer_base_class, nil)
+      end
+
+      def concurrency
+        options.fetch(:concurrency, nil)
+      end
+
+      def params_batch_transformation
+        options.fetch(:params_batch_transformation,
+          Dionysus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy.new)
+      end
+    end
+  end
+end