dionysus-rb 0.1.0

data/lib/dionysus/utils/null_tracer.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullTracer
+  def self.trace(_event_name, _topic); end
+end

data/lib/dionysus/utils/null_transaction_provider.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::NullTransactionProvider
+  def self.transaction
+    yield
+  end
+
+  def self.connection_pool
+    self
+  end
+
+  def self.with_connection
+    yield
+  end
+end

data/lib/dionysus/utils/sidekiq_batched_job_distributor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class Dionysus::Utils::SidekiqBatchedJobDistributor
+  attr_reader :batch_size, :units_count, :time_range_in_seconds
+  private     :batch_size, :units_count, :time_range_in_seconds
+
+  def initialize(batch_size:, units_count:, time_range_in_seconds:)
+    @batch_size = batch_size
+    @units_count = units_count
+    @time_range_in_seconds = time_range_in_seconds
+  end
+
+  def number_of_batches
+    (units_count.to_d / batch_size).ceil
+  end
+
+  def time_per_batch
+    (time_range_in_seconds.to_d / number_of_batches).floor
+  end
+
+  def enqueue_batch(job_class, queue, batch_number, *job_arguments)
+    job_class.set(queue: queue).perform_in(batch_number * time_per_batch, *job_arguments)
+  end
+end

data/lib/dionysus/utils.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Dionysus::Utils
+  autoload :KarafkaDatadogListener, "dionysus/utils/karafka_datadog_listener"
+  autoload :KarafkaSentryListener, "dionysus/utils/karafka_sentry_listener"
+end

data/lib/dionysus/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Dionysus
+  module Version
+  end
+  VERSION = "0.1.0"
+end

data/lib/dionysus-rb.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "dionysus"

data/lib/dionysus.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "logger"
+require "sidekiq"
+require "sidekiq-cron"
+require "karafka"
+require "dry-monitor"
+require "concurrent/array"
+require "file-based-healthcheck"
+require "sigurd"
+require "securerandom"
+require "zeitwerk"
+
+module Dionysus
+  CONSUMER_GROUP_PREFIX = "dionysus_consumer_group_for"
+
+  def self.loader
+    @loader ||= Zeitwerk::Loader.for_gem.tap do |loader|
+      loader.ignore("#{__dir__}/dionysus-rb.rb")
+    end
+  end
+
+  def self.initialize_application!(environment:, seed_brokers:, client_id:, logger:, draw_routing: true, consumer_group_prefix: CONSUMER_GROUP_PREFIX)
+    ENV["KARAFKA_ENV"] = environment
+
+    karafka_app = Class.new(Karafka::App) do
+      setup do |config|
+        config.kafka = {
+          "bootstrap.servers": seed_brokers.join(","),
+          "client.id": client_id
+        }
+        config.client_id = client_id
+        config.logger = logger
+        yield config if block_given?
+      end
+    end
+
+    Object.const_set(:KarafkaApp, karafka_app)
+    self.karafka_application = karafka_app
+    evaluate_routing(consumer_group_prefix: consumer_group_prefix) if consumer_registry.present? && draw_routing
+  end
+
+  def self.karafka_application=(karafka_app)
+    @karafka_application = karafka_app
+  end
+
+  def self.karafka_application
+    @karafka_application
+  end
+
+  def self.health_check=(health_check)
+    @health_check = health_check
+
+    Karafka.monitor.subscribe("app.initialized") do |_event|
+      health_check = Dionysus.health_check
+
+      health_check&.app_initialized!
+    end
+
+    Karafka.monitor.subscribe("statistics.emitted") do |_event|
+      health_check = Dionysus.health_check
+
+      health_check&.register_heartbeat
+    end
+
+    Karafka.monitor.subscribe("consumer.consumed") do |_event|
+      health_check = Dionysus.health_check
+
+      health_check&.register_heartbeat
+    end
+
+    Karafka.monitor.subscribe("app.stopped") do |_event|
+      health_check = Dionysus.health_check
+
+      health_check&.app_stopped!
+    end
+  end
+
+  def self.health_check
+    @health_check
+  end
+
+  def self.outbox_worker_health_check
+    @outbox_worker_health_check ||= Dionysus::Producer::Outbox::HealthCheck.new
+  end
+
+  def self.enable_outbox_worker_healthcheck
+    monitor.subscribe("outbox_producer.started") { outbox_worker_health_check.register_heartbeat }
+    monitor.subscribe("outbox_producer.stopped") { outbox_worker_health_check.worker_stopped }
+    monitor.subscribe("outbox_producer.heartbeat") { outbox_worker_health_check.register_heartbeat }
+  end
+
+  def self.logger
+    if karafka_application
+      karafka_application.config.logger
+    else
+      Logger.new($stdout)
+    end
+  end
+
+  def self.consumer_registry
+    @consumer_registry
+  end
+
+  def self.inject_routing!(registry)
+    @consumer_registry = registry
+  end
+
+  def self.monitor
+    @monitor ||= Dionysus::Monitor.new
+  end
+
+  def self.evaluate_routing(consumer_group_prefix:)
+    consumer_group_name = "#{consumer_group_prefix}_#{karafka_application.config.client_id}"
+    karafka_application.instance_exec(consumer_registry) do |registry|
+      consumer_groups.draw do
+        consumer_group consumer_group_name do
+          registry.registrations.each do |_, registration|
+            registration.topics.each do |topic|
+              topic topic.to_s do
+                consumer topic.consumer
+                instance_eval(&topic.extensions_block) if topic.extensions_block
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+  private_class_method :evaluate_routing
+end
+
+Dionysus.loader.setup

data/lib/tasks/dionysus.rake

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+namespace :dionysus do
+  desc "Starts the Dionysus RB Outbox Producer"
+  task producer: :environment do
+    $stdout.sync = true
+    Rails.logger.info("Running dionysus:producer rake task.")
+    threads_number = ENV.fetch("DIONYSUS_RB_THREADS_NUMBER", 1).to_i
+    Dionysus::Producer.start_outbox_worker(threads_number: threads_number)
+  end
+
+  desc "Validates config for :attributes for observables"
+  task validate_columns: :environment do
+    $stdout.sync = true
+    Rails.logger.info("Running dionysus:validate_columns rake task.")
+    Dionysus::Producer::Registry::Validator.new.validate_columns
+  end
+end

data/sig/dionysus/rb.rbs

@@ -0,0 +1,6 @@
+module Dionysus
+  module Rb
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end