dionysus-rb 0.1.0

data/lib/dionysus/producer/partition_key.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::PartitionKey
+  attr_reader :resource, :config
+  private     :resource, :config
+
+  def initialize(resource, config: Dionysus::Producer.configuration)
+    @resource = resource
+    @config = config
+  end
+
+  def to_key(responder:)
+    if has_custom_partition_key?(responder)
+      apply_custom_partition_key(responder)
+    else
+      apply_default_partition_key
+    end
+  end
+
+  private
+
+  def has_custom_partition_key?(responder)
+    responder.partition_key.present?
+  end
+
+  def apply_custom_partition_key(responder)
+    apply_partition_key(responder.partition_key)
+  end
+
+  def apply_default_partition_key
+    apply_partition_key(config.default_partition_key)
+  end
+
+  def apply_partition_key(partition_key)
+    if partition_key.respond_to?(:call)
+      resolved_partition_key = partition_key.call(resource)
+      resolved_partition_key&.to_s
+    elsif resource.respond_to?(partition_key)
+      resource.public_send(partition_key).to_i.to_s
+    end
+  end
+end

data/lib/dionysus/producer/registry/validator.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Registry::Validator
+  attr_reader :registry
+  private :registry
+
+  def initialize(registry: Dionysus::Producer.registry)
+    @registry = registry
+  end
+
+  def validate_columns
+    registry.registrations.each_value do |registration|
+      registration.topics.each do |topic|
+        topic
+          .models
+          .flat_map(&:observables_config)
+          .each { |observable_config| validate_observable(observable_config) }
+      end
+    end
+  end
+
+  private
+
+  def validate_observable(observable_config)
+    model = observable_config.fetch(:model)
+    attributes = observable_config.fetch(:attributes)
+
+    return if attributes.all? { |attribute| model.column_names.include?(attribute.to_s) }
+
+    raise ArgumentError.new("some attributes #{attributes} do not exist on model #{model}")
+  end
+end

data/lib/dionysus/producer/registry.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Registry
+  attr_reader :container
+  private :container
+
+  def initialize
+    @container = {}
+  end
+
+  def namespace(namespace, &block)
+    registration = Registration.new(namespace)
+    registration.instance_eval(&block)
+    container[namespace] = registration
+  end
+
+  def registrations
+    container
+  end
+
+  class Registration
+    attr_reader :namespace, :topics, :serializer_klass, :producers
+
+    def initialize(namespace)
+      @namespace = namespace
+      @topics = []
+      @serializer_klass = nil
+      @producers = []
+    end
+
+    def serializer(serializer_klass)
+      @serializer_klass = serializer_klass
+    end
+
+    def topic(name, options = {}, &block)
+      new_topic = Topic.new(namespace, name, serializer_klass, options)
+      new_topic.instance_eval(&block)
+      producer = Dionysus::Producer::KarafkaResponderGenerator.new.generate(
+        Dionysus::Producer.configuration, new_topic
+      )
+      producers << producer
+      new_topic.producer = producer
+      topics << new_topic
+    end
+
+    class Topic
+      GENESIS_SUFFIX = "genesis"
+      private_constant :GENESIS_SUFFIX
+
+      attr_reader :namespace, :name, :serializer_klass, :options, :models
+
+      attr_accessor :producer
+
+      def initialize(namespace, name, serializer_klass, options = {})
+        @namespace = namespace
+        @name = name
+        @options = options
+        @serializer_klass = serializer_klass
+        @models = []
+      end
+
+      def to_s
+        Dionysus::TopicName.new(namespace, name).to_s
+      end
+
+      def genesis_to_s
+        Dionysus::TopicName.new(namespace, "#{name}_#{GENESIS_SUFFIX}").to_s if genesis_replica?
+      end
+
+      def partition_key
+        options.fetch(:partition_key, nil)
+      end
+
+      def genesis_replica?
+        options.fetch(:genesis_replica, false) == true
+      end
+
+      def publish(model_klass, model_registrations_options = {})
+        @models << ModelRegistration.new(model_klass, model_registrations_options)
+      end
+
+      def publishes_model?(model_klass)
+        models.any? { |registration| registration.model_klass == model_klass }
+      end
+
+      class ModelRegistration
+        attr_reader :model_klass, :options
+
+        def initialize(model_klass, options = {})
+          @model_klass = model_klass
+          @options = options
+          validate_and_set_up
+        end
+
+        def observes?(resource, changeset)
+          observer_config_for(resource, changeset).present?
+        end
+
+        def association_name_for_observable(resource, changeset)
+          observer_config_for(resource, changeset).fetch(:association_name)
+        end
+
+        def observables_config
+          options.fetch(:observe, [])
+        end
+
+        private
+
+        def validate_and_set_up
+          validate_options
+          set_up_as_publishables
+        end
+
+        def validate_options
+          return unless options.key?(:observe)
+
+          options[:observe].each do |observer_config|
+            model = observer_config.fetch(:model)
+            association_name = observer_config.fetch(:association_name)
+            _attributes = observer_config.fetch(:attributes)
+
+            if association_name.is_a?(Symbol) && !model.instance_methods.include?(association_name.to_sym)
+              raise ArgumentError.new("association name :#{association_name} does not exist on model #{model}")
+            end
+          end
+        end
+
+        def set_up_as_publishables
+          [model_klass, dependencies_from_parent, observables]
+            .flatten
+            .select { |model| model.instance_of?(Class) && model.ancestors.include?(ActiveRecord::Base) }
+            .each { |model| ensure_model_is_publishable(model) }
+        end
+
+        def dependencies_from_parent
+          options.fetch(:with, [])
+        end
+
+        def observables
+          options.fetch(:observe, []).map { |hash| hash.fetch(:model) }
+        end
+
+        def ensure_model_is_publishable(model)
+          model.include(publishable_module) unless model.included_modules.include?(publishable_module)
+        end
+
+        def publishable_module
+          Dionysus::Producer::Outbox::ActiveRecordPublishable
+        end
+
+        def observer_config_for(resource, changeset)
+          resource_model_name = resource.model_name.to_s
+          changeset_attributes = changeset.keys.map(&:to_sym)
+
+          options[:observe].to_a.find do |observer_config|
+            config_model_name = observer_config.fetch(:model).to_s
+            config_attributes = observer_config.fetch(:attributes).to_a.map(&:to_sym)
+
+            config_model_name == resource_model_name && (config_attributes & changeset_attributes).any?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dionysus/producer/serializer.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Serializer
+  def self.serialize(record_or_records, dependencies: [])
+    new(record_or_records, dependencies: dependencies).serialize
+  end
+
+  attr_reader :records, :dependencies
+  private     :records, :dependencies
+
+  def initialize(record_or_records, dependencies: [])
+    @records = Array.wrap(record_or_records).compact
+
+    @dependencies = dependencies
+  end
+
+  def serialize
+    records.map do |record|
+      serializer = resolve_serializer_for_record(record)
+      serializer.as_json
+    end
+  end
+
+  private
+
+  def resolve_serializer_for_record(record)
+    if record.persisted?
+      infer_serializer.new(record, include: include, context_serializer: self.class)
+    else
+      deleted_record_serializer(record)
+    end
+  end
+
+  def infer_serializer
+    raise "implement me!"
+  end
+
+  def deleted_record_serializer(record)
+    Dionysus::Producer::DeletedRecordSerializer.new(record, include: include,
+      context_serializer: self.class)
+  end
+
+  def model_klass
+    records.first&.class
+  end
+
+  def include
+    dependencies.to_a.map do |model_klass|
+      [model_klass.model_name.plural.to_sym, model_klass.model_name.singular.to_sym]
+    end.flatten
+  end
+end

data/lib/dionysus/producer/suppressor.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Suppressor
+  SUPPRESSION_KEY = :dionysus_producer_suppressed
+  private_constant :SUPPRESSION_KEY
+
+  def self.suppressed?
+    Thread.current[SUPPRESSION_KEY] == true
+  end
+
+  def self.suppress!
+    Thread.current[SUPPRESSION_KEY] = true
+  end
+
+  def self.unsuppress!
+    Thread.current[SUPPRESSION_KEY] = nil
+  end
+end

data/lib/dionysus/producer.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer
+  def self.configuration
+    @configuration ||= Dionysus::Producer::Config.new
+  end
+
+  def self.configure
+    yield configuration
+  end
+
+  def self.registry
+    configuration.registry
+  end
+
+  def self.declare(&config)
+    registry = Dionysus::Producer::Registry.new
+
+    registry.instance_eval(&config)
+    configure do |configuration|
+      configuration.registry = registry
+    end
+  end
+
+  def self.outbox
+    Dionysus::Producer::Outbox.new(configuration.outbox_model, config: configuration)
+  end
+
+  def self.outbox_publisher
+    Dionysus::Producer::Outbox::Publisher.new(config: configuration)
+  end
+
+  def self.reset!
+    return if registry.nil?
+
+    registry.registrations.values.flat_map(&:producers).each do |producer_class|
+      Dionysus.send(:remove_const, producer_class.name.demodulize.to_sym) if producer_class.name
+    end
+    @configuration = Dionysus::Producer::Config.new
+  end
+
+  def self.responders_for(model_klass)
+    return [] if registry.nil?
+
+    registry.registrations.each.with_object([]) do |(_, registration), responders|
+      registration.producers.select { |producer| producer.publisher_of?(model_klass) }.each do |producer|
+        responders << producer
+      end
+    end
+  end
+
+  def self.responders_for_model_for_topic(model_klass, topic)
+    responders_for(model_klass).select { |responder| responder.publisher_of_model_for_topic?(model_klass, topic) }
+  end
+
+  def self.responders_for_dependency_parent(model_klass)
+    return [] if registry.nil?
+
+    registry.registrations.values.each.with_object([]) do |registration, accum|
+      registration.topics.each do |topic|
+        topic
+          .models
+          .select { |model_registration| model_registration.options[:with].to_a.include?(model_klass) }
+          .each do |model_registration|
+          accum << [model_registration.model_klass, topic.producer]
+        end
+      end
+    end
+  end
+
+  def self.responders_for_dependency_parent_for_topic(model_klass, topic)
+    responders_for_dependency_parent(model_klass).select do |_model, responder|
+      responder.publisher_for_topic?(topic)
+    end
+  end
+
+  def self.start_outbox_worker(threads_number:)
+    runners = (1..threads_number).map do
+      Dionysus::Producer::Outbox::Runner.new(config: configuration)
+    end
+    executor = Sigurd::Executor.new(runners, sleep_seconds: 5, logger: Dionysus.logger)
+    signal_handler = Sigurd::SignalHandler.new(executor)
+    signal_handler.run!
+  end
+
+  def self.topics_models_mapping
+    return {} if registry.nil?
+
+    registry
+      .registrations
+      .values
+      .flat_map(&:topics)
+      .to_h do |topic|
+        [
+          topic.to_s,
+          topic.models.to_h { |registration| [registration.model_klass, registration.options.fetch(:with, [])] }
+        ]
+      end
+  end
+
+  def self.observers_with_responders_for(resource, changeset)
+    return [] if registry.nil?
+
+    registry.registrations.values.each.with_object([]) do |registration, accum|
+      registration.topics.each do |topic|
+        topic
+          .models
+          .select { |model_registration| model_registration.observes?(resource, changeset) }
+          .each do |model_registration|
+            association_name = model_registration.association_name_for_observable(resource, changeset)
+            methods_chain = association_name.to_s.split(".")
+            association_or_associations = methods_chain.inject(resource) do |record, method_name|
+              record.public_send(method_name)
+            end
+
+            accum << [Array.wrap(association_or_associations).compact, topic.producer]
+          end
+      end
+    end
+  end
+end

data/lib/dionysus/railtie.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+if defined?(Rails)
+  class Dionysus::Railtie < Rails::Railtie
+    rake_tasks do
+      load "tasks/dionysus.rake"
+    end
+  end
+end

data/lib/dionysus/rb/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Dionysus::Rb
+  VERSION = "0.1.0"
+end

data/lib/dionysus/rb.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "rb/version"
+
+# Your code goes here...
+class Dionysus::Rb::Error < StandardError
+  # Your code goes here...
+end

data/lib/dionysus/support/rspec/outbox_publishable.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+RSpec.shared_examples_for "Dionysus Transactional Outbox Publishable" do |extra_attributes|
+  let(:attributes_for_model) { extra_attributes.to_h }
+
+  describe "Dionysus Transactional Outbox" do
+    let(:outbox_model) { Dionysus::Producer.configuration.outbox_model }
+
+    before do
+      outbox_model.delete_all
+    end
+
+    context "when the record gets created" do
+      subject(:create_record) { create(described_class.model_name.singular.to_sym, attributes_for_model) }
+
+      let(:resource_class) { described_class.model_name.to_s }
+      let(:resource_id) { create_record.id }
+      let(:event_name) { "#{described_class.model_name.singular}_created" }
+      let(:outbox_records_for_resource) do
+        outbox_model.where(resource_class: resource_class, event_name: event_name)
+      end
+      let(:outbox_record_for_created_resource) do
+        outbox_model.find_by(resource_class: resource_class, resource_id: resource_id, event_name: event_name)
+      end
+
+      it "creates an Outbox Record" do
+        expect do
+          create_record
+        end.to change { outbox_records_for_resource.count }.from(0)
+        expect(outbox_record_for_created_resource).to be_present
+      end
+    end
+
+    context "when the record gets updated" do
+      subject(:update_record) { record.update(attributes_for_model.merge(updated_at: Time.current)) }
+
+      let!(:record) { create(described_class.model_name.singular.to_sym) }
+      let(:resource_class) { described_class.model_name.to_s }
+      let(:resource_id) { record.id }
+      let(:event_name) { "#{described_class.model_name.singular}_updated" }
+      let(:outbox_records_for_resource) do
+        outbox_model.where(resource_class: resource_class, event_name: event_name)
+      end
+      let(:outbox_record_for_updated_resource) do
+        outbox_model.find_by(resource_class: resource_class, resource_id: resource_id, event_name: event_name)
+      end
+
+      it "creates an Outbox Record" do
+        expect do
+          update_record
+        end.to change { outbox_records_for_resource.count }.from(0)
+        expect(outbox_record_for_updated_resource).to be_present
+      end
+    end
+
+    context "when the record gets deleted" do
+      subject(:delete_record) { record.destroy }
+
+      let!(:record) { create(described_class.model_name.singular.to_sym) }
+      let(:resource_class) { described_class.model_name.to_s }
+      let(:resource_id) { record.id }
+      let(:event_name) { "#{described_class.model_name.singular}_destroyed" }
+      let(:outbox_records_for_resource) do
+        outbox_model.where(resource_class: resource_class, event_name: event_name)
+      end
+      let(:outbox_record_for_deleted_resource) do
+        outbox_model.find_by(resource_class: resource_class, resource_id: resource_id, event_name: event_name)
+      end
+
+      it "creates an Outbox Record" do
+        expect do
+          delete_record
+        end.to change { outbox_records_for_resource.count }.from(0)
+        expect(outbox_record_for_deleted_resource).to be_present
+      end
+    end
+  end
+end

data/lib/dionysus/topic_name.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class Dionysus::TopicName
+  attr_reader :namespace, :name
+  private     :namespace, :name
+
+  def initialize(namespace, name)
+    @namespace = namespace
+    @name = name
+  end
+
+  def to_s
+    "#{namespace}_#{name}"
+  end
+end

data/lib/dionysus/utils/default_message_filter.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::DefaultMessageFilter
+  attr_reader :error_handler
+  private     :error_handler
+
+  def initialize(error_handler:)
+    @error_handler = error_handler
+  end
+
+  def ignore_message?(topic:, message:, transformed_data:)
+    false
+  end
+
+  def notify_about_ignored_message(topic:, message:, transformed_data:)
+    error_handler.capture_message(error_message(topic, message, transformed_data))
+  end
+
+  private
+
+  def error_message(topic, message, transformed_data)
+    "Ignoring Kafka message. Make sure it's processed later (e.g. by directly doing it from console): " \
+      "topic: #{topic}, message: #{message}, transformed_data: #{transformed_data}."
+  end
+end

data/lib/dionysus/utils/exponential_backoff.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::ExponentialBackoff
+  def self.backoff_for(multiplier, count)
+    (multiplier * (2**count))
+  end
+end

data/lib/dionysus/utils/karafka_datadog_listener.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::KarafkaDatadogListener
+  class << self
+    def on_error_occurred(event)
+      span = tracer.active_span
+      span&.set_error(event[:error])
+    end
+
+    private
+
+    def tracer
+      if Datadog.respond_to?(:tracer)
+        Datadog.tracer
+      else
+        Datadog::Tracing
+      end
+    end
+  end
+end

data/lib/dionysus/utils/karafka_sentry_listener.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Dionysus::Utils::KarafkaSentryListener
+  class << self
+    def on_error_occurred(event)
+      Sentry.capture_exception(event[:error])
+    end
+  end
+end

data/lib/dionysus/utils/null_error_handler.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullErrorHandler
+  def self.capture_exception(_error); end
+  def self.capture_message(_error); end
+end

data/lib/dionysus/utils/null_event_bus.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullEventBus
+  def self.publish(_name, _payload); end
+end

data/lib/dionysus/utils/null_hermes_event_producer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullHermesEventProducer
+  def self.publish(*); end
+end

data/lib/dionysus/utils/null_instrumenter.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullInstrumenter
+  def self.instrument(_name, _payload = {})
+    yield
+  end
+end

data/lib/dionysus/utils/null_lock_client.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullLockClient
+  def self.lock(resource_key, expiration_time)
+    payload = {
+      validity: expiration_time,
+      resource: resource_key,
+      value: "null_lock_client_lock"
+    }
+
+    yield payload
+  end
+end

data/lib/dionysus/utils/null_model_factory.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullModelFactory
+  def self.for_model(_model_name); end
+end

data/lib/dionysus/utils/null_mutex_provider.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullMutexProvider
+  def self.with_lock(_name)
+    yield
+  end
+end

data/lib/dionysus/utils/null_retry_provider.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullRetryProvider
+  def self.retry
+    yield
+  end
+end