lepus 0.0.1.beta2

data/lib/lepus/producer.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Lepus
+  class Producer
+    DEFAULT_EXCHANGE_OPTIONS = {
+      type: :topic,
+      durable: true,
+      auto_delete: false
+    }.freeze
+
+    DEFAULT_PUBLISH_OPTIONS = {
+      expiration: 7 * (60 * 60 * 24)
+    }.freeze
+
+    def initialize(exchange_name, **options)
+      @exchange_name = exchange_name
+      @exchange_options = DEFAULT_EXCHANGE_OPTIONS.merge(options)
+    end
+
+    def publish(message, **options)
+      payload = if message.is_a?(String)
+        options[:content_type] ||= "text/plain"
+        message
+      else
+        options[:content_type] ||= "application/json"
+        MultiJson.dump(message)
+      end
+
+      bunny.with_channel do |channel|
+        exchange = channel.exchange(@exchange_name, @exchange_options)
+        exchange.publish(
+          payload,
+          DEFAULT_PUBLISH_OPTIONS.merge(options)
+        )
+      end
+    end
+
+    def bunny
+      Thread.current[:lepus_bunny] ||= Lepus.config.create_connection(suffix: "producer")
+    end
+  end
+end

data/lib/lepus/rails/log_subscriber.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+class Lepus::LogSubscriber < ActiveSupport::LogSubscriber
+  def start_process(event)
+    process = event.payload[:process]
+
+    attributes = {
+      pid: process.pid,
+      hostname: process.hostname,
+      process_id: process.process_id,
+      name: process.name
+    }.merge(process.metadata)
+
+    info formatted_event(event, action: "Started #{process.kind}", **attributes)
+  end
+
+  def shutdown_process(event)
+    process = event.payload[:process]
+
+    attributes = {
+      pid: process.pid,
+      hostname: process.hostname,
+      process_id: process.process_id,
+      name: process.name
+    }.merge(process.metadata)
+
+    info formatted_event(event, action: "Shutdown #{process.kind}", **attributes)
+  end
+
+  def register_process(event)
+    process_kind = event.payload[:kind]
+    attributes = event.payload.slice(:pid, :hostname, :process_id, :name)
+
+    if (error = event.payload[:error])
+      warn formatted_event(event, action: "Error registering #{process_kind}", **attributes.merge(error: formatted_error(error)))
+    else
+      debug formatted_event(event, action: "Register #{process_kind}", **attributes)
+    end
+  end
+
+  def deregister_process(event)
+    process = event.payload[:process]
+
+    attributes = {
+      process_id: process.id,
+      pid: process.pid,
+      hostname: process.hostname,
+      name: process.name,
+      last_heartbeat_at: process.last_heartbeat_at&.strftime("%Y-%m-%d %H:%M:%S"),
+      claimed_size: event.payload[:claimed_size],
+      pruned: event.payload[:pruned]
+    }
+
+    if (error = event.payload[:error])
+      warn formatted_event(event, action: "Error deregistering #{process.kind}", **attributes.merge(error: formatted_error(error)))
+    else
+      debug formatted_event(event, action: "Deregister #{process.kind}", **attributes)
+    end
+  end
+
+  def prune_processes(event)
+    debug formatted_event(event, action: "Prune dead processes", **event.payload.slice(:size))
+  end
+
+  def thread_error(event)
+    error formatted_event(event, action: "Error in thread", error: formatted_error(event.payload[:error]))
+  end
+
+  def graceful_termination(event)
+    attributes = event.payload.slice(:process_id, :supervisor_pid, :supervised_processes)
+
+    if event.payload[:shutdown_timeout_exceeded]
+      warn formatted_event(event, action: "Supervisor wasn't terminated gracefully - shutdown timeout exceeded", **attributes)
+    else
+      info formatted_event(event, action: "Supervisor terminated gracefully", **attributes)
+    end
+  end
+
+  def immediate_termination(event)
+    info formatted_event(event, action: "Supervisor terminated immediately", **event.payload.slice(:process_id, :supervisor_pid, :supervised_processes))
+  end
+
+  def unhandled_signal_error(event)
+    error formatted_event(event, action: "Received unhandled signal", **event.payload.slice(:signal))
+  end
+
+  def replace_fork(event)
+    status = event.payload[:status]
+    attributes = event.payload.slice(:pid).merge \
+      status: status.exitstatus || "no exit status set",
+      pid_from_status: status.pid,
+      signaled: status.signaled?,
+      stopsig: status.stopsig,
+      termsig: status.termsig
+
+    if (replaced_fork = event.payload[:fork])
+      info formatted_event(event, action: "Replaced terminated #{replaced_fork.kind}", **attributes.merge(hostname: replaced_fork.hostname, name: replaced_fork.name))
+    else
+      warn formatted_event(event, action: "Tried to replace forked process but it had already died", **attributes)
+    end
+  end
+
+  private
+
+  def formatted_event(event, action:, **attributes)
+    "Lepus-#{Lepus::VERSION} #{action} (#{event.duration.round(1)}ms)  #{formatted_attributes(**attributes)}"
+  end
+
+  def formatted_attributes(**attributes)
+    attributes.map { |attr, value| "#{attr}: #{value.inspect}" }.join(", ")
+  end
+
+  def formatted_error(error)
+    [error.class, error.message].compact.join(" ")
+  end
+
+  def logger
+    Lepus.logger
+  end
+end

data/lib/lepus/rails/railtie.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Lepus
+  class Railtie < ::Rails::Railtie
+    config.lepus = ActiveSupport::OrderedOptions.new
+    config.lepus.app_executor = nil
+    config.lepus.on_thread_error = nil
+
+    initializer "lepus.app_executor", before: :run_prepare_callbacks do |app|
+      config.lepus.app_executor ||= app.executor
+      if ::Rails.respond_to?(:error) && config.lepus.on_thread_error.nil?
+        config.lepus.on_thread_error = ->(exception) { ::Rails.error.report(exception, handled: false) }
+      elsif config.lepus.on_thread_error.nil?
+        config.lepus.on_thread_error = ->(exception) { Lepus.logger.error(exception) }
+      end
+
+      Lepus.config.app_executor = config.lepus.app_executor
+      Lepus.config.on_thread_error = config.lepus.on_thread_error
+    end
+
+    initializer "lepus.logger" do
+      ActiveSupport.on_load(:lepus) do
+        self.logger = ::Rails.logger if logger == Lepus::DEFAULT_LOGGER
+      end
+
+      Lepus::LogSubscriber.attach_to :lepus
+    end
+  end
+
+  ActiveSupport.run_load_hooks(:lepus, self)
+end

data/lib/lepus/rails.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+require "active_support/log_subscriber"
+
+require_relative "rails/log_subscriber"
+require_relative "rails/railtie"

data/lib/lepus/supervisor/config.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "logger"
+
+module Lepus
+  class Supervisor < Processes::Base
+    class Config
+      class ProcessStruct < Struct.new(:process_class, :attributes)
+        def instantiate
+          process_class.new(**attributes)
+        end
+      end
+
+      attr_accessor :pidfile, :require_file
+
+      def initialize(require_file: nil, pidfile: "tmp/pids/lepus.pid", **kwargs)
+        @pidfile = pidfile
+        @require_file = require_file
+        self.consumers = kwargs[:consumers] if kwargs.key?(:consumers)
+      end
+
+      def configured_processes
+        consumer_processes
+      end
+
+      def consumers=(vals)
+        @consumer_processes = nil
+        @consumers = Array(vals).map(&:to_s)
+      end
+
+      def consumers
+        @consumers ||= Lepus::Consumer.descendants.reject(&:abstract_class?).map(&:name).compact
+      end
+
+      protected
+
+      def consumer_processes
+        @consumer_processes ||= consumers.map do |class_name|
+          ProcessStruct.new(Lepus::Processes::Consumer, {class_name: class_name})
+        end
+      end
+    end
+  end
+end

data/lib/lepus/supervisor/maintenance.rb

@@ -0,0 +1,35 @@
+module Lepus
+  class Supervisor < Processes::Base
+    module Maintenance
+      def self.included(base)
+        base.send :include, InstanceMethods
+      end
+
+      module InstanceMethods
+        private
+
+        def launch_maintenance_task
+          @maintenance_task = ::Concurrent::TimerTask.new(run_now: true, execution_interval: Lepus.config.process_alive_threshold) do
+            prune_dead_processes
+          end
+
+          @maintenance_task.add_observer do |_, _, error|
+            handle_thread_error(error) if error
+          end
+
+          @maintenance_task.execute
+        end
+
+        def stop_maintenance_task
+          @maintenance_task&.shutdown
+        end
+
+        def prune_dead_processes
+          wrap_in_app_executor do
+            Lepus::Process.prune(excluding: process)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lepus/supervisor/pidfile.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Lepus
+  class Supervisor < Processes::Base
+    class Pidfile
+      def initialize(path)
+        @path = path
+        @pid = ::Process.pid
+      end
+
+      def setup
+        check_status
+        write_file
+        set_at_exit_hook
+      end
+
+      def delete
+        delete_file
+      end
+
+      private
+
+      attr_reader :path, :pid
+
+      def check_status
+        if ::File.exist?(path)
+          existing_pid = ::File.read(path).strip.to_i
+          existing_pid > 0 && ::Process.kill(0, existing_pid)
+
+          already_running!
+        else
+          FileUtils.mkdir_p File.dirname(path)
+        end
+      rescue Errno::ESRCH
+        # Process is dead, ignore, just delete the file
+        delete
+      rescue Errno::EPERM
+        already_running!
+      end
+
+      def write_file
+        ::File.open(path, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |file| file.write(pid.to_s) }
+      rescue Errno::EEXIST
+        check_status
+        retry
+      end
+
+      def set_at_exit_hook
+        at_exit { delete if ::Process.pid == pid }
+      end
+
+      def delete_file
+        ::File.delete(path) if ::File.exist?(path)
+      end
+
+      def already_running!
+        abort "A supervisor is already running. Check #{path}"
+      end
+    end
+  end
+end

data/lib/lepus/supervisor/pidfiled.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Lepus
+  class Supervisor < Processes::Base
+    module Pidfiled
+      def self.included(base)
+        base.send :include, InstanceMethods
+        base.class_eval do
+          before_boot :setup_pidfile
+          after_shutdown :delete_pidfile
+        end
+      end
+
+      module InstanceMethods
+        private
+
+        def setup_pidfile
+          if (path = configuration.pidfile)
+            @pidfile = Pidfile.new(path).tap(&:setup)
+          end
+        end
+
+        def delete_pidfile
+          @pidfile&.delete
+        end
+      end
+    end
+  end
+end

data/lib/lepus/supervisor/signals.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Lepus
+  class Supervisor < Processes::Base
+    module Signals
+      def self.included(base)
+        base.send :include, InstanceMethods
+        base.class_eval do
+          before_boot :register_signal_handlers
+          after_shutdown :restore_default_signal_handlers
+        end
+      end
+
+      module InstanceMethods
+        private
+
+        SIGNALS = %i[QUIT INT TERM]
+
+        def register_signal_handlers
+          SIGNALS.each do |signal|
+            trap(signal) do
+              signal_queue << signal
+              interrupt
+            end
+          end
+        end
+
+        def restore_default_signal_handlers
+          SIGNALS.each do |signal|
+            trap(signal, :DEFAULT)
+          end
+        end
+
+        def process_signal_queue
+          while (signal = signal_queue.shift)
+            handle_signal(signal)
+          end
+        end
+
+        def handle_signal(signal)
+          case signal
+          when :TERM, :INT
+            stop
+            terminate_gracefully
+          when :QUIT
+            stop
+            terminate_immediately
+          else
+            Lepus.instrument :unhandled_signal_error, signal: signal
+          end
+        end
+
+        def signal_processes(pids, signal)
+          pids.each do |pid|
+            signal_process pid, signal
+          end
+        end
+
+        def signal_process(pid, signal)
+          ::Process.kill signal, pid
+        rescue Errno::ESRCH
+          # Ignore, process died before
+        end
+
+        def signal_queue
+          @signal_queue ||= []
+        end
+      end
+    end
+  end
+end

data/lib/lepus/supervisor.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+module Lepus
+  class Supervisor < Processes::Base
+    include LifecycleHooks
+    include Maintenance
+    include Signals
+    include Pidfiled
+
+    class << self
+      def start(**options)
+        # Lepus.config.supervisor = true
+        config = Config.new(**options)
+        new(config).tap(&:start)
+      end
+    end
+
+    def initialize(configuration)
+      @configuration = configuration
+      @forks = {}
+      @configured_processes = {}
+      ProcessRegistry.instance # Ensure the registry is initialized
+    end
+
+    def start
+      boot
+
+      run_start_hooks
+
+      start_processes
+      launch_maintenance_task
+
+      supervise
+    end
+
+    def stop
+      super
+      run_stop_hooks
+    end
+
+    private
+
+    attr_reader :configuration, :forks, :configured_processes
+
+    def boot
+      Lepus.instrument(:start_process, process: self) do
+        if configuration.require_file
+          Kernel.require configuration.require_file
+        else
+          begin
+            require "rails"
+            require_relative "rails"
+            require File.expand_path("config/environment", Dir.pwd)
+          rescue LoadError
+            # Rails not found
+          end
+        end
+
+        setup_consumers
+        check_bunny_connection
+
+        run_process_callbacks(:boot) do
+          sync_std_streams
+        end
+      end
+    end
+
+    def setup_consumers
+      Lepus.eager_load_consumers!
+
+      if configuration.consumers.empty?
+        abort "No consumers found. Exiting..."
+      end
+    end
+
+    def check_bunny_connection
+      temp_bunny = Lepus.config.create_connection(suffix: "(boot-check)")
+      temp_bunny.close
+    end
+
+    def start_processes
+      configuration.configured_processes.each do |configured_process|
+        start_process(configured_process)
+      end
+    end
+
+    def supervise
+      loop do
+        break if stopped?
+
+        set_procline
+        process_signal_queue
+
+        unless stopped?
+          reap_and_replace_terminated_forks
+          interruptible_sleep(1)
+        end
+      end
+    ensure
+      shutdown
+    end
+
+    def start_process(configured_process)
+      process_instance = configured_process.instantiate.tap do |instance|
+        instance.supervised_by process
+        instance.mode = :fork
+      end
+
+      process_instance.before_fork
+      pid = fork do
+        process_instance.after_fork
+        process_instance.start
+      end
+
+      configured_processes[pid] = configured_process
+      forks[pid] = process_instance
+    end
+
+    def set_procline
+      procline "supervising #{supervised_processes.join(", ")}"
+    end
+
+    def terminate_gracefully
+      Lepus.instrument(:graceful_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: supervised_processes) do |payload|
+        term_forks
+
+        shutdown_timeout = 5
+        puts "\nWaiting up to #{shutdown_timeout} seconds for processes to terminate gracefully..."
+        Timer.wait_until(shutdown_timeout, -> { all_forks_terminated? }) do
+          reap_terminated_forks
+        end
+
+        unless all_forks_terminated?
+          payload[:shutdown_timeout_exceeded] = true
+          terminate_immediately
+        end
+      end
+    end
+
+    def terminate_immediately
+      Lepus.instrument(:immediate_termination, process_id: process_id, supervisor_pid: ::Process.pid, supervised_processes: supervised_processes) do
+        quit_forks
+      end
+    end
+
+    def shutdown
+      Lepus.instrument(:shutdown_process, process: self) do
+        run_process_callbacks(:shutdown) do
+          stop_maintenance_task
+        end
+      end
+    end
+
+    def sync_std_streams
+      $stdout.sync = $stderr.sync = true
+    end
+
+    def supervised_processes
+      forks.keys
+    end
+
+    def term_forks
+      signal_processes(forks.keys, :TERM)
+    end
+
+    def quit_forks
+      signal_processes(forks.keys, :QUIT)
+    end
+
+    def reap_and_replace_terminated_forks
+      loop do
+        pid, status = ::Process.waitpid2(-1, ::Process::WNOHANG)
+        break unless pid
+
+        replace_fork(pid, status)
+      end
+    end
+
+    def reap_terminated_forks
+      loop do
+        pid, _ = ::Process.waitpid2(-1, ::Process::WNOHANG)
+        break unless pid
+
+        configured_processes.delete(pid)
+      end
+    rescue SystemCallError
+      # All children already reaped
+    end
+
+    def replace_fork(pid, status)
+      Lepus.instrument(:replace_fork, supervisor_pid: ::Process.pid, pid: pid, status: status) do |payload|
+        if (terminated_fork = forks.delete(pid))
+          payload[:fork] = terminated_fork
+
+          start_process(configured_processes.delete(pid))
+        end
+      end
+    end
+
+    def all_forks_terminated?
+      forks.empty?
+    end
+  end
+end

data/lib/lepus/timer.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Timer
+    extend self
+
+    def wait_until(timeout, condition, &block)
+      if timeout > 0
+        deadline = monotonic_time_now + timeout
+
+        while monotonic_time_now < deadline && !condition.call
+          sleep 0.1
+          yield
+        end
+      else
+        until condition.call
+          sleep 0.5
+          yield
+        end
+      end
+    end
+
+    private
+
+    def monotonic_time_now
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/lepus/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Lepus
+  VERSION = "0.0.1.beta2"
+end