dionysus-rb 0.1.0

data/lib/dionysus/producer/karafka_responder_generator.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/string"
+
+class Dionysus::Producer::KarafkaResponderGenerator
+  TOMBSTONE = nil
+
+  def generate(config, topic)
+    topic_name = topic.to_s
+    genesis_topic_name = topic.genesis_to_s if topic.genesis_replica?
+
+    responder_klass = Class.new(Dionysus::Producer::BaseResponder) do
+      topic topic_name
+      topic genesis_topic_name if topic.genesis_replica?
+
+      define_method :respond do |batch, options = {}|
+        config.instrumenter.instrument("dionysus.respond.#{self.class.name}") do
+          final_options = {}
+          if (partition_key = options.fetch(:partition_key, nil))
+            final_options[:partition_key] = partition_key
+          end
+          if (key = options.fetch(:key, nil))
+            final_options[:key] = key
+          end
+
+          if genesis_only?(options) && genesis_topic_name.nil?
+            raise "cannot execute genesis-only as there is no genesis topic for responder #{self.class.name}"
+          end
+
+          if batch.nil?
+            unless genesis_only?(options)
+              respond_to topic_name, TOMBSTONE, **final_options
+              config.event_bus.publish("dionysus.respond", topic_name: topic_name, message: TOMBSTONE,
+                options: final_options)
+            end
+            if topic.genesis_replica?
+              respond_to genesis_topic_name, TOMBSTONE, **final_options
+              config.event_bus.publish("dionysus.respond", topic_name: genesis_topic_name, message: TOMBSTONE,
+                options: final_options)
+            end
+          else
+            message = Array.wrap(batch).map do |event, record_or_records, batch_options|
+              records = Array.wrap(record_or_records)
+              return if records.empty?
+
+              record = records.sample
+
+              payload = serialize_to_payload(records, topic, batch_options)
+
+              {
+                event: event,
+                model_name: record.model_name.name,
+                data: payload
+              }
+            end
+            unless genesis_only?(options)
+              respond_to topic_name, { message: message }, **final_options
+              config.event_bus.publish("dionysus.respond", topic_name: topic_name, message: message,
+                options: final_options)
+            end
+            if topic.genesis_replica?
+              respond_to genesis_topic_name, { message: message }, **final_options
+              config.event_bus.publish("dionysus.respond", topic_name: genesis_topic_name, message: message,
+                options: final_options)
+            end
+          end
+        end
+      end
+
+      private
+
+      define_method :serialize_to_payload do |records, current_topic, batch_options|
+        if batch_options.to_h[:serialize] == false
+          records.map(&:as_json)
+        else
+          record = records.sample
+
+          model_klass = record.class
+          dependencies = current_topic
+            .models
+            .find(-> { NullRegistration.new }) { |model_registration| model_registration.model_klass == model_klass }
+            .options
+            .to_h
+            .fetch(:with, [])
+
+          current_topic.serializer_klass.serialize(records, dependencies: dependencies)
+        end
+      end
+
+      define_method :genesis_only? do |options|
+        options.fetch(:genesis_only, false) == true
+      end
+    end
+
+    responder_klass.instance_exec(topic) do |dionysus_topic|
+      define_singleton_method :publisher_of? do |model_klass|
+        dionysus_topic.publishes_model?(model_klass)
+      end
+
+      define_singleton_method :publisher_for_topic? do |current_topic|
+        if dionysus_topic.genesis_replica?
+          dionysus_topic.to_s == current_topic.to_s || dionysus_topic.genesis_to_s == current_topic.to_s
+        else
+          dionysus_topic.to_s == current_topic.to_s
+        end
+      end
+
+      define_singleton_method :publisher_of_model_for_topic? do |model_klass, current_topic|
+        dionysus_topic.publishes_model?(model_klass) && publisher_for_topic?(current_topic)
+      end
+
+      define_singleton_method :partition_key do
+        dionysus_topic.partition_key
+      end
+
+      define_singleton_method :primary_topic do
+        responder_klass.topics.values.first
+      end
+    end
+
+    responder_klass_name = "#{topic.to_s.classify}Responder"
+
+    Dionysus.send(:remove_const, responder_klass_name) if Dionysus.const_defined?(responder_klass_name)
+    Dionysus.const_set(responder_klass_name, responder_klass)
+    responder_klass
+  end
+
+  class NullRegistration
+    def options
+      {}
+    end
+  end
+end

data/lib/dionysus/producer/key.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Key
+  attr_reader :resource
+  private :resource
+
+  def initialize(resource)
+    @resource = resource
+  end
+
+  def to_key
+    "#{resource.model_name}:#{resource.id}"
+  end
+end

data/lib/dionysus/producer/model_serializer.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::ModelSerializer
+  attr_reader :record, :include, :context_serializer
+  private     :record, :include, :context_serializer
+
+  def initialize(record, include:, context_serializer:)
+    @record = record
+    @include = include
+    @context_serializer = context_serializer
+  end
+
+  def self.attributes(*names)
+    Array(names).each do |name|
+      attribute(name, {})
+    end
+  end
+
+  def self.attribute(name, options = {})
+    declared_attributes << [name, options]
+
+    define_method(name) do
+      record.public_send(name)
+    end
+  end
+
+  def self.has_one(name, options = {})
+    declared_to_one_relationships << [name, options]
+
+    define_method(name) do
+      record.public_send(name)
+    end
+
+    define_method("#{name}_id") do
+      record.public_send("#{name}_id")
+    end
+  end
+
+  def self.has_many(name, options = {})
+    declared_to_many_relationships << [name, options]
+
+    define_method(name) do
+      record.public_send(name)
+    end
+
+    define_method("#{name.to_s.singularize}_ids") do
+      record.public_send("#{name.to_s.singularize}_ids")
+    end
+  end
+
+  def self.declared_attributes
+    @declared_attributes ||= []
+  end
+
+  def self.declared_to_one_relationships
+    @declared_to_one_relationships ||= []
+  end
+
+  def self.declared_to_many_relationships
+    @declared_to_many_relationships ||= []
+  end
+
+  def as_json
+    {}.tap do |payload|
+      declared_attributes.each do |declared_attribute, _options|
+        payload[declared_attribute] = send(declared_attribute)
+      end
+      payload["links"] = {}
+      declared_to_one_relationships.each do |declared_relationship, _options|
+        payload["links"][declared_relationship] = send("#{declared_relationship}_id")
+      end
+      declared_to_many_relationships.each do |declared_relationship, _options|
+        payload["links"][declared_relationship] = send("#{declared_relationship.to_s.singularize}_ids")
+      end
+
+      include.each do |relationship_to_include|
+        relationship_to_include = relationship_to_include.to_sym
+
+        if declared_to_one_relationships.to_h.key?(relationship_to_include)
+          payload[relationship_to_include] =
+            context_serializer.serialize(send(relationship_to_include), dependencies: []).first
+        end
+
+        if declared_to_many_relationships.to_h.key?(relationship_to_include)
+          payload[relationship_to_include] =
+            context_serializer.serialize(send(relationship_to_include), dependencies: [])
+        end
+      end
+    end.deep_stringify_keys
+  end
+
+  private
+
+  def declared_attributes
+    self.class.declared_attributes
+  end
+
+  def declared_to_one_relationships
+    self.class.declared_to_one_relationships
+  end
+
+  def declared_to_many_relationships
+    self.class.declared_to_many_relationships
+  end
+end

data/lib/dionysus/producer/outbox/active_record_publishable.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module Dionysus::Producer::Outbox::ActiveRecordPublishable
+  extend ActiveSupport::Concern
+
+  OUTBOX_RECORDS_TO_PUBLISH_STORAGE_KEY = :bookingsync_outbox_records_to_publish
+  private_constant :OUTBOX_RECORDS_TO_PUBLISH_STORAGE_KEY
+
+  included do
+    after_create :dionysus_insert_model_created
+    after_update :dionysus_insert_model_updated
+    after_destroy :dionysus_insert_model_destroy
+    after_commit :publish_outbox_records
+
+    def self.outbox_records_to_publish
+      Thread.current[OUTBOX_RECORDS_TO_PUBLISH_STORAGE_KEY] ||= Concurrent::Array.new
+    end
+
+    def self.add_outbox_records_to_publish(records)
+      outbox_records_to_publish.concat(records)
+    end
+
+    def self.clear_records_to_publish
+      Thread.current[OUTBOX_RECORDS_TO_PUBLISH_STORAGE_KEY] = Concurrent::Array.new
+    end
+  end
+
+  def dionysus_publish_updates_after_soft_delete?
+    false
+  end
+
+  private
+
+  def dionysus_insert_model_created
+    add_outbox_records_to_publish(Dionysus::Producer.outbox.insert_created(self))
+  end
+
+  def dionysus_insert_model_updated
+    add_outbox_records_to_publish(Dionysus::Producer.outbox.insert_updated(self))
+  end
+
+  def dionysus_insert_model_destroy
+    add_outbox_records_to_publish(Dionysus::Producer.outbox.insert_destroyed(self))
+  end
+
+  def add_outbox_records_to_publish(records)
+    self.class.add_outbox_records_to_publish(records)
+  end
+
+  def publish_outbox_records
+    begin
+      if publish_after_commit?
+        records_processor.call(
+          self.class.outbox_records_to_publish.sort_by(&:resource_created_at)
+        )
+      end
+    rescue => e
+      Dionysus.logger.error(
+        "[Dionysus Outbox from publish_outbox_records] #{e.class}: #{e}"
+      )
+    end
+    self.class.clear_records_to_publish
+  end
+
+  def records_processor
+    Dionysus::Producer::Outbox::RecordsProcessor.new
+  end
+
+  def publish_after_commit?
+    Dionysus::Producer.configuration.publish_after_commit
+  end
+end

data/lib/dionysus/producer/outbox/datadog_latency_reporter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::DatadogLatencyReporter
+  attr_reader :config
+  private     :config
+
+  def initialize(config: Dionysus::Producer.configuration)
+    @config = config
+  end
+
+  def report(latency: generate_latency)
+    datadog_statsd_client.gauge("dionysus.producer.outbox.latency.minimum", latency.minimum)
+    datadog_statsd_client.gauge("dionysus.producer.outbox.latency.maximum", latency.maximum)
+    datadog_statsd_client.gauge("dionysus.producer.outbox.latency.average", latency.average)
+    datadog_statsd_client.gauge("dionysus.producer.outbox.latency.highest_since_creation_date",
+      latency.highest_since_creation_date)
+  end
+
+  private
+
+  delegate :datadog_statsd_client, :datadog_statsd_prefix, to: :config
+
+  def generate_latency
+    Dionysus::Producer::Outbox::LatencyTracker.new.calculate
+  end
+end

data/lib/dionysus/producer/outbox/datadog_latency_reporter_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::DatadogLatencyReporterJob
+  include Sidekiq::Worker
+
+  sidekiq_options queue: Dionysus::Producer::Config.high_priority_sidekiq_queue
+
+  def perform
+    Dionysus::Producer::Outbox::DatadogLatencyReporter.new.report
+  end
+end

data/lib/dionysus/producer/outbox/datadog_latency_reporter_scheduler.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::DatadogLatencyReporterScheduler
+  JOB_NAME = "dionysus_producer_outbox_datadog_latency_reporter_job"
+  EVERY_MINUTE_IN_CRON_SYNTAX = "* * * * *"
+  JOB_CLASS_NAME = "Dionysus::Producer::Outbox::DatadogLatencyReporterJob"
+  JOB_DESCRIPTION = "Collect latency metrics from dionysus outbox and send them to Datadog"
+
+  private_constant :JOB_NAME, :EVERY_MINUTE_IN_CRON_SYNTAX, :JOB_CLASS_NAME, :JOB_DESCRIPTION
+
+  attr_reader :config
+  private     :config
+
+  def initialize(config: Dionysus::Producer.configuration)
+    @config = config
+  end
+
+  def add_to_schedule
+    find || create
+  end
+
+  private
+
+  def find
+    Sidekiq::Cron::Job.find(name: JOB_NAME)
+  end
+
+  def create
+    Sidekiq::Cron::Job.create(create_job_arguments)
+  end
+
+  def create_job_arguments
+    {
+      name: JOB_NAME,
+      cron: EVERY_MINUTE_IN_CRON_SYNTAX,
+      class: JOB_CLASS_NAME,
+      queue: config.high_priority_sidekiq_queue,
+      active_job: false,
+      description: JOB_DESCRIPTION,
+      date_as_argument: false
+    }
+  end
+
+  def every_minute_to_cron_syntax
+    EVERY_MINUTE_IN_CRON_SYNTAX
+  end
+end

data/lib/dionysus/producer/outbox/datadog_tracer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::DatadogTracer
+  SERVICE_NAME = "dionysus_outbox_worker"
+  private_constant :SERVICE_NAME
+
+  def self.service_name
+    SERVICE_NAME
+  end
+
+  def trace(event_name, topic)
+    tracer.trace(event_name, span_type: "worker", service: self.class.service_name, on_error: error_handler) do |span|
+      span.set_tag("topic", topic)
+
+      yield
+    end
+  end
+
+  private
+
+  def tracer
+    if Datadog.respond_to?(:tracer)
+      Datadog.tracer
+    else
+      Datadog::Tracing
+    end
+  end
+
+  def error_handler
+    ->(span, error) { span.set_error(error) }
+  end
+end

data/lib/dionysus/producer/outbox/duplicates_filter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::DuplicatesFilter
+  def self.call(records_to_publish)
+    new(records_to_publish).call
+  end
+
+  attr_reader :records_to_publish
+  private     :records_to_publish
+
+  def initialize(records_to_publish)
+    @records_to_publish = records_to_publish
+  end
+
+  def call
+    records_to_publish
+      .slice_when { |record_1, record_2| generate_uniqueness_key(record_1) != generate_uniqueness_key(record_2) }
+      .flat_map(&:last)
+  end
+
+  private
+
+  def generate_uniqueness_key(record)
+    [record.resource_class, record.resource_id, record.event_name, record.topic]
+  end
+end

data/lib/dionysus/producer/outbox/event_name.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::EventName
+  attr_reader :resource_name
+  private :resource_name
+
+  def initialize(resource_name)
+    @resource_name = resource_name
+  end
+
+  def created
+    "#{resource_name}_created"
+  end
+
+  def updated
+    "#{resource_name}_updated"
+  end
+
+  def destroyed
+    "#{resource_name}_destroyed"
+  end
+
+  def for_event_type(type)
+    public_send(type)
+  end
+end

data/lib/dionysus/producer/outbox/health_check.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::HealthCheck
+  KEY_PREFIX = "__dionysus_outbox_worker__running__"
+  TMP_DIR = "/tmp"
+  private_constant :KEY_PREFIX, :TMP_DIR
+
+  def self.check(hostname: ENV.fetch("HOSTNAME", nil), expiry_time_in_seconds: 120)
+    new(hostname: hostname, expiry_time_in_seconds: expiry_time_in_seconds).check
+  end
+
+  attr_reader :hostname, :expiry_time_in_seconds
+
+  def initialize(hostname: ENV.fetch("HOSTNAME", nil), expiry_time_in_seconds: 120)
+    @hostname = hostname
+    @expiry_time_in_seconds = expiry_time_in_seconds
+  end
+
+  def check
+    if healthcheck_storage.running?
+      ""
+    else
+      "[Dionysus Producer Outbox healthcheck failed]"
+    end
+  end
+
+  def register_heartbeat
+    healthcheck_storage.touch
+  end
+
+  def worker_stopped
+    healthcheck_storage.remove
+  end
+
+  private
+
+  def healthcheck_storage
+    @healthcheck_storage ||= FileBasedHealthcheck.new(
+      directory: TMP_DIR,
+      filename: key,
+      time_threshold: expiry_time_in_seconds
+    )
+  end
+
+  def key
+    "#{KEY_PREFIX}#{hostname}"
+  end
+end

data/lib/dionysus/producer/outbox/latency_tracker.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::LatencyTracker
+  LatencyTrackerResult = Struct.new(:minimum, :maximum, :average, :highest_since_creation_date)
+  private_constant :LatencyTrackerResult
+
+  attr_reader :config, :clock
+  private     :config, :clock
+
+  def initialize(config: Dionysus::Producer.configuration, clock: Time)
+    @config = config
+    @clock = clock
+  end
+
+  def calculate(interval: 1.minute)
+    records = outbox_model.published_since(interval.ago)
+    latencies = records.map(&:publishing_latency)
+
+    LatencyTrackerResult.new(
+      latencies.min.to_d,
+      latencies.max.to_d,
+      calculate_average(latencies),
+      calculate_highest_since_creation_date.to_d
+    )
+  end
+
+  private
+
+  delegate :outbox_model, to: :config
+
+  def calculate_average(latencies)
+    if latencies.any?
+      latencies.sum.to_d / latencies.size
+    else
+      0
+    end
+  end
+
+  def calculate_highest_since_creation_date
+    minimum_created_at_from_not_published = outbox_model.not_published.minimum(:created_at) or return 0
+    clock.current - minimum_created_at_from_not_published
+  end
+end