dionysus-rb 0.1.0

data/lib/dionysus/producer/outbox/model.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Dionysus::Producer::Outbox::Model
+  extend ActiveSupport::Concern
+
+  OBSERVER_TOPIC = "__outbox_observer__"
+  CHANGESET_COLUMN = "changeset"
+  private_constant :OBSERVER_TOPIC, :CHANGESET_COLUMN
+
+  def self.observer_topic
+    OBSERVER_TOPIC
+  end
+
+  included do
+    scope :fetch_publishable, lambda { |batch_size, topic|
+      outbox_worker_publishing_delay = Dionysus::Producer.configuration.outbox_worker_publishing_delay
+
+      records = where(published_at: nil, topic: topic)
+        .where("retry_at IS NULL OR retry_at <= ?", Time.current)
+        .order(created_at: :asc)
+        .limit(batch_size)
+      if outbox_worker_publishing_delay > 0.seconds
+        records = records.where("created_at <= ?", Time.current + outbox_worker_publishing_delay)
+      end
+      records
+    }
+    scope :published_since, ->(time) { where("published_at >= ?", time) }
+    scope :not_published, -> { where(published_at: nil) }
+
+    belongs_to :resource, polymorphic: true, foreign_type: :resource_class, optional: true
+
+    def self.pending_topics
+      not_published.select("DISTINCT topic").map(&:topic)
+    end
+
+    def self.handles_changeset?
+      column_names.include?(CHANGESET_COLUMN)
+    end
+
+    def self.encrypts_changeset!
+      define_method :changeset= do |payload|
+        super(payload.to_json)
+      end
+    end
+  end
+
+  def observer?
+    topic == Dionysus::Producer::Outbox::Model.observer_topic
+  end
+
+  def transformed_changeset
+    return {} unless self.class.handles_changeset?
+
+    if changeset.respond_to?(:to_hash)
+      changeset.symbolize_keys
+    else
+      JSON.parse(changeset).symbolize_keys
+    end
+  end
+
+  def published?
+    published_at.present?
+  end
+
+  def failed?
+    failed_at.present?
+  end
+
+  def handle_error(raised_error, clock: Time)
+    @error = raised_error
+    self.error_class = raised_error.class
+    self.error_message = raised_error.message
+    self.failed_at = clock.current
+    self.attempts ||= 0
+    self.attempts += 1
+    self.retry_at = clock.current.advance(seconds: Dionysus::Utils::ExponentialBackoff.backoff_for(5,
+      attempts))
+  end
+
+  def error
+    if error_class_arity == 1 || error_class_arity == -1
+      error_class.constantize.new(error_message)
+    else
+      StandardError.new("#{error_class_constant}: #{error_message}")
+    end
+  end
+
+  def resource_created_at
+    return created_at if resource.nil?
+
+    resource.created_at
+  end
+
+  def publishing_latency
+    return unless published?
+
+    published_at - created_at
+  end
+
+  def created_event?
+    event_name.to_s.end_with?("created")
+  end
+
+  def updated_event?
+    event_name.to_s.end_with?("updated")
+  end
+
+  private
+
+  def error_class_constant
+    error_class.constantize
+  end
+
+  def error_class_arity
+    error_class_constant.instance_method(:initialize).arity
+  end
+end

data/lib/dionysus/producer/outbox/producer.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::Producer
+  attr_reader :config
+  private :config
+
+  def initialize(config: Dionysus::Producer.configuration)
+    @config = config
+  end
+
+  def call(topic, batch_size: config.outbox_publishing_batch_size)
+    outbox_model.fetch_publishable(batch_size, topic).to_a.tap do |records_to_publish|
+      records_processor.call(records_to_publish) do |record|
+        yield record if block_given?
+      end
+    end
+  end
+
+  private
+
+  delegate :outbox_model, to: :config
+
+  def records_processor
+    @records_processor ||= Dionysus::Producer::Outbox::RecordsProcessor.new
+  end
+end

data/lib/dionysus/producer/outbox/publishable.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::Publishable < SimpleDelegator
+  attr_reader :model, :config, :soft_delete_column
+  private     :config, :soft_delete_column
+
+  def initialize(model, config: Dionysus::Producer.configuration)
+    @model = model
+    @config = config
+    @soft_delete_column = config.soft_delete_column.to_s
+    super(model)
+  end
+
+  def model_class
+    model.class
+  end
+
+  def primary_key_attribute
+    model.class.primary_key
+  end
+
+  def publishable_id
+    model.public_send(primary_key_attribute)
+  end
+
+  def model_name
+    model.class.model_name
+  end
+
+  def resource_name
+    model_name.singular
+  end
+
+  def previously_changed?
+    previous_changes.present?
+  end
+
+  def previous_changes_include_canceled?
+    !!previous_changes[soft_delete_column]
+  end
+
+  def previous_changes_uncanceled?
+    previous_changes_include_canceled? && previous_changes[soft_delete_column][0].present? && visible?
+  end
+
+  def previous_changes_canceled?
+    previous_changes_include_canceled? && previous_changes[soft_delete_column][0].blank? && soft_deleted?
+  end
+
+  def previous_changes_still_canceled?
+    previous_changes_include_canceled? && previous_changes[soft_delete_column][0].present? && soft_deleted?
+  end
+
+  # TODO: Check if this is needed
+  def previous_changed_still_visible?
+    previous_changes_include_canceled? && previous_changes[soft_delete_column][0].blank? && visible?
+  end
+
+  def soft_deleted?
+    public_send(soft_delete_column).present?
+  end
+
+  def soft_deletable?
+    model.respond_to?(soft_delete_column)
+  end
+
+  def visible?
+    !soft_deleted?
+  end
+
+  def topics
+    top_level_topics = Dionysus::Producer
+      .responders_for(model.class)
+      .map(&:primary_topic)
+    topics_from_dependencies = Dionysus::Producer
+      .responders_for_dependency_parent(model.class)
+      .map(&:last)
+      .map(&:primary_topic)
+
+    [top_level_topics, topics_from_dependencies]
+      .flatten
+      .uniq
+      .tap { |standard_topics| standard_topics << observer_topic if add_observer_topic? }
+  end
+
+  def changeset
+    if model.destroyed?
+      {
+        model.class.primary_key => [model.public_send(model.class.primary_key), nil],
+        "created_at" => [model.created_at, nil]
+      }
+    else
+      model.previous_changes
+    end
+  end
+
+  private
+
+  delegate :observer_topic, to: Dionysus::Producer::Outbox::Model
+  delegate :observers_with_responders_for, to: Dionysus::Producer
+  delegate :outbox_model, to: :config
+
+  def add_observer_topic?
+    outbox_model.handles_changeset? && observers_with_responders_for(self, changeset).any?
+  end
+end

data/lib/dionysus/producer/outbox/publisher.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::Publisher
+  attr_reader :config
+  private :config
+
+  def initialize(config:)
+    @config = config
+  end
+
+  def publish(outbox_record, options = {})
+    return if Dionysus::Producer::Suppressor.suppressed?
+
+    instrument("publishing_with_dionysus") do
+      resource_class = outbox_record.resource_class.constantize
+      primary_key = resource_class.primary_key
+      primary_key_value = outbox_record.resource_id
+      topic = outbox_record.topic
+      resource = resource_class.find_by(primary_key => primary_key_value) ||
+        resource_class.new(primary_key => primary_key_value)
+      event_name = outbox_record.event_name
+      if resource.new_record? && outbox_record.created_event?
+        logger.error(
+          "Attempted to publish #{resource.class}, id: #{resource.id} but it was deleted, that should never happen!"
+        )
+        return
+      end
+
+      if resource.new_record? && outbox_record.updated_event?
+        logger.error(
+          "There was an update of #{resource.class}, id: #{resource.id} but it was deleted, that should never happen!"
+        )
+        return
+      end
+
+      publish_for_top_level_resource(outbox_record, resource, event_name, topic, options)
+      publish_for_dependency(resource, topic, options)
+    end
+  end
+
+  def publish_observers(outbox_record)
+    return if Dionysus::Producer::Suppressor.suppressed?
+
+    instrument("publishing_observers_with_dionysus") do
+      resource_class = outbox_record.resource_class.constantize
+      primary_key = resource_class.primary_key
+      primary_key_value = outbox_record.resource_id
+      resource = resource_class.find_by(primary_key => primary_key_value) ||
+        resource_class.new(primary_key => primary_key_value)
+      changeset = outbox_record.transformed_changeset
+
+      Dionysus::Producer.observers_with_responders_for(resource,
+        changeset).each do |observers, responder|
+        if observers.count > config.observers_inline_maximum_size
+          execute_genesis_for_observers(observers, responder)
+        else
+          observers.each { |observer_record| publish_observer(observer_record, responder) }
+        end
+      end
+    end
+  end
+
+  private
+
+  delegate :instrumenter, :error_handler, to: :config
+  delegate :instrument, to: :instrumenter
+  delegate :logger, to: Dionysus
+
+  def publish_for_top_level_resource(outbox_record, resource, event_name, topic, options)
+    Dionysus::Producer.responders_for_model_for_topic(resource.class, topic).each do |responder|
+      partition_key = outbox_record.partition_key.presence || Dionysus::Producer::PartitionKey.new(
+        resource
+      ).to_key(responder: responder)
+      key = Dionysus::Producer::Key.new(resource).to_key
+
+      responder.call(generate_message(event_name, resource), options.merge(partition_key: partition_key, key: key))
+    end
+  end
+
+  def publish_for_dependency(resource, topic, options)
+    Dionysus::Producer.responders_for_dependency_parent_for_topic(resource.class,
+      topic).each do |parent_klass, responder|
+      parent_event_name = Dionysus::Producer::Outbox::EventName.new(
+        parent_klass.model_name.singular
+      ).updated
+
+      if resource.class.reflect_on_association(parent_klass.model_name.singular)
+        parent_records = [resource.public_send(parent_klass.model_name.singular)]
+      elsif resource.class.reflect_on_association(parent_klass.model_name.plural)
+        parent_records = resource.public_send(parent_klass.model_name.plural)
+      else
+        next
+      end
+
+      example_parent_record = parent_records.first or next
+      partition_key = Dionysus::Producer::PartitionKey.new(example_parent_record, config: config)
+        .to_key(responder: responder)
+      key = Dionysus::Producer::Key.new(example_parent_record).to_key
+
+      parent_records.each do |parent_record|
+        responder.call(generate_message(parent_event_name, parent_record),
+          options.merge(partition_key: partition_key, key: key))
+      end
+    end
+  end
+
+  def publish_observer(observer_record, responder)
+    publishable = Dionysus::Producer::Outbox::Publishable.new(observer_record)
+    partition_key = Dionysus::Producer::PartitionKey.new(publishable).to_key(responder: responder)
+    key = Dionysus::Producer::Key.new(publishable).to_key
+    event_name = Dionysus::Producer::Outbox::EventName.new(publishable.resource_name).updated
+
+    responder.call(generate_message(event_name, publishable), partition_key: partition_key, key: key)
+  end
+
+  def generate_message(event_name, resource)
+    [[event_name, resource, {}]]
+  end
+
+  def execute_genesis_for_observers(observers, responder)
+    resource_class = observers.first.class
+    primary_key = resource_class.primary_key
+
+    Dionysus::Producer::Genesis::Streamer::StandardJob.enqueue(
+      resource_class.where(primary_key => observers),
+      resource_class,
+      responder.primary_topic,
+      number_of_days: 0.1
+    )
+  end
+end

data/lib/dionysus/producer/outbox/records_processor.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::RecordsProcessor
+  attr_reader :config
+  private :config
+
+  def initialize(config: Dionysus::Producer.configuration)
+    @config = config
+  end
+
+  def call(records)
+    failed_records = []
+    records_to_publish = resolve_records_to_publish(records)
+    records_to_publish.each do |record|
+      begin
+        publish(record)
+      rescue => e
+        record.handle_error(e)
+        record.save!
+        failed_records << record
+      end
+      yield record if block_given?
+    end
+    published_records = records - failed_records
+    mark_as_published(published_records)
+    records
+  end
+
+  private
+
+  delegate :outbox_model, to: :config
+
+  def resolve_records_to_publish(records)
+    return records unless config.remove_consecutive_duplicates_before_publishing
+
+    Dionysus::Producer::Outbox::DuplicatesFilter.call(records)
+  end
+
+  def publish(record)
+    if record.observer?
+      outbox_publisher.publish_observers(record)
+    else
+      outbox_publisher.publish(record)
+    end
+  end
+
+  def outbox_publisher
+    @outbox_publisher ||= Dionysus::Producer.outbox_publisher
+  end
+
+  def mark_as_published(published_records)
+    outbox_model
+      .where(id: published_records)
+      .update_all(published_at: Time.current, error_class: nil, error_message: nil, failed_at: nil, retry_at: nil)
+  end
+end

data/lib/dionysus/producer/outbox/runner.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::Runner
+  attr_reader :logger, :id, :config
+  private     :logger, :config
+
+  def initialize(config: Dionysus::Producer.configuration,
+    logger: Dionysus.logger)
+    @id = SecureRandom.uuid
+    @logger = logger
+    logger.push_tags("Dionysus::Producer::Outbox::Runners #{id}") if logger.respond_to?(:push_tags)
+    @config = config
+  end
+
+  def start
+    log("started")
+    instrument("outbox_producer.started")
+    @should_stop = false
+    ensure_database_connection!
+    loop do
+      if @should_stop
+        instrument("outbox_producer.shutting_down")
+        log("shutting down")
+        break
+      end
+      process_topics
+      instrument("outbox_producer.heartbeat")
+      sleep outbox_worker_sleep_seconds
+    end
+  rescue => e
+    error_handler.capture_exception(e)
+    log("error: #{e} #{e.message}")
+    instrument("outbox_producer.error", error: e, error_message: e.message)
+    raise e
+  end
+
+  def stop
+    log("Outbox worker stopping")
+    instrument("outbox_producer.stopped")
+    @should_stop = true
+  end
+
+  private
+
+  delegate :outbox_worker_sleep_seconds, :database_connection_provider, :outbox_model,
+    :lock_client, :lock_expiry_time, :error_handler,
+    :outbox_publishing_batch_size, to: :config
+
+  delegate :pending_topics, to: :outbox_model
+  delegate :lock, to: :lock_client
+
+  def process_topics
+    pending_topics.each do |topic|
+      instrument("outbox_producer.processing_topic", topic: topic) do
+        tracer.trace("outbox_producer", topic) do
+          lock(lock_name(topic), lock_expiry_time) do |locked|
+            if locked
+              producer.call(topic, batch_size: outbox_publishing_batch_size) do |record|
+                if record.failed?
+                  instrument("outbox_producer.publishing_failed", outbox_record: record)
+                  error("failed to publish #{record.inspect}")
+                  error_handler.capture_exception(record.error)
+                else
+                  debug("published #{record.inspect}")
+                  instrument("outbox_producer.published", outbox_record: record)
+                end
+              end
+              instrument("outbox_producer.processed_topic", topic: topic)
+            else
+              debug("lock exists for #{topic} topic")
+              instrument("outbox_producer.lock_exists_for_topic", topic: topic)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def ensure_database_connection!
+    database_connection_provider.connection.reconnect!
+  end
+
+  def log(message)
+    logger.info("#{log_prefix} #{message}")
+  end
+
+  def debug(message)
+    logger.debug("#{log_prefix} #{message}")
+  end
+
+  def error(message)
+    logger.error("#{log_prefix} #{message}")
+  end
+
+  def lock_name(topic)
+    "dionysus_#{topic}_lock"
+  end
+
+  def log_prefix
+    "[Dionysus] Outbox worker"
+  end
+
+  def instrument(*args, **kwargs)
+    Dionysus.monitor.instrument(*args, **kwargs) do
+      yield if block_given?
+    end
+  end
+
+  def tracer
+    @tracer ||= if Object.const_defined?(:Datadog)
+      Dionysus::Producer::Outbox::DatadogTracer.new
+    else
+      Dionysus::Utils::NullTracer
+    end
+  end
+
+  def producer
+    @producer ||= Dionysus::Producer::Outbox::Producer.new
+  end
+end

data/lib/dionysus/producer/outbox/tombstone_publisher.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox::TombstonePublisher
+  TOMBSTONE = nil
+  private_constant :TOMBSTONE
+
+  attr_reader :config
+  private :config
+
+  def initialize(config: Dionysus::Producer.configuration)
+    @config = config
+  end
+
+  def tombstone(resource, responder, options = {})
+    partition_key = options.fetch(:partition_key) do
+      Dionysus::Producer::PartitionKey.new(resource, config: config).to_key(responder: responder)
+    end
+    key = options.fetch(:key) { Dionysus::Producer::Key.new(resource).to_key }
+
+    responder.call(TOMBSTONE, partition_key: partition_key, key: key)
+  end
+end

data/lib/dionysus/producer/outbox.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+class Dionysus::Producer::Outbox
+  attr_reader :outbox_model, :config
+  private     :outbox_model, :config
+
+  def initialize(outbox_model, config:)
+    @outbox_model = outbox_model
+    @config = config
+  end
+
+  def insert_created(record)
+    insert(Publishable.new(record), :created)
+  end
+
+  def insert_updated(record)
+    publishable = Publishable.new(record)
+    return [] unless publishable.previously_changed?
+
+    if publishable.respond_to?(config.soft_delete_column)
+      event_type = event_type_for_update_of_soft_deletable_record(publishable) or return []
+      insert(publishable, event_type)
+    else
+      insert(publishable, :updated)
+    end
+  end
+
+  def insert_destroyed(record)
+    publishable = Publishable.new(record)
+
+    insert(publishable, :destroyed)
+  end
+
+  private
+
+  delegate :transaction_provider, :transactional_outbox_enabled, to: :config
+
+  def insert(publishable, event_type)
+    return [] unless transactional_outbox_enabled
+
+    transaction_provider.transaction do
+      publishable.topics.map do |topic|
+        attributes = {
+          resource: publishable.model,
+          event_name: event_name_for(publishable, event_type),
+          partition_key: partition_key_for_publishable_for_topic(publishable, topic),
+          topic: topic
+        }
+        attributes[:changeset] = publishable.changeset if observer_topic?(topic)
+
+        outbox_model.create!(attributes)
+      end
+    end
+  end
+
+  def partition_key_for_publishable_for_topic(publishable, topic)
+    responder = Dionysus::Producer
+      .responders_for_model_for_topic(publishable.model_class, topic)
+      .first or return
+
+    Dionysus::Producer::PartitionKey.new(publishable).to_key(responder: responder)
+  end
+
+  def observer_topic?(topic)
+    topic == Dionysus::Producer::Outbox::Model.observer_topic
+  end
+
+  def event_name_for(publishable, event_type)
+    Dionysus::Producer::Outbox::EventName
+      .new(publishable.resource_name)
+      .for_event_type(event_type)
+  end
+
+  def event_type_for_update_of_soft_deletable_record(publishable)
+    if publishable.previous_changes_include_canceled?
+      event_type_for_update_of_soft_deletable_record_for_soft_delete_state_change(publishable)
+    else
+      event_type_for_update_of_soft_deletable_record_for_standard_state_change(publishable)
+    end
+  end
+
+  def event_type_for_update_of_soft_deletable_record_for_soft_delete_state_change(publishable)
+    if publishable.previous_changes_uncanceled?
+      :created
+    elsif publishable.previous_changes_canceled?
+      :destroyed
+    elsif publishable.previous_changes_still_canceled? || publishable.previous_changed_still_visible?
+      nil
+    else
+      raise "that should never happen"
+    end
+  end
+
+  def event_type_for_update_of_soft_deletable_record_for_standard_state_change(publishable)
+    if publishable.visible? || (publishable.soft_deleted? && publishable.dionysus_publish_updates_after_soft_delete?)
+      :updated
+    elsif publishable.soft_deleted?
+      nil
+    else
+      raise "that should never happen"
+    end
+  end
+end