lepus 0.0.1.beta2

data/lib/lepus/process.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Lepus
+  class Process
+    class NotFoundError < RuntimeError
+      def initialize(id)
+        super("Process with id #{id} not found")
+      end
+    end
+
+    ATTRIBUTES = %i[id name pid hostname kind last_heartbeat_at supervisor_id].freeze
+    MEMORY_GRABBER = case RUBY_PLATFORM
+    when /linux/
+      ->(pid) {
+        IO.readlines("/proc/#{$$}/status").each do |line|
+          next unless line.start_with?("VmRSS:")
+          break line.split[1].to_i
+        end
+      }
+    when /darwin|bsd/
+      ->(pid) {
+        `ps -o pid,rss -p #{pid}`.lines.last.split.last.to_i
+      }
+    else
+      ->(pid) { 0 }
+    end
+
+    class << self
+      def register(**attributes)
+        attributes[:id] ||= SecureRandom.uuid
+        Lepus.instrument :register_process, **attributes do |payload|
+          new(**attributes).tap do |process|
+            ProcessRegistry.instance.add(process)
+            payload[:process_id] = process.id
+          end
+        rescue Exception => error # rubocop:disable Lint/RescueException
+          payload[:error] = error
+          raise
+        end
+      end
+
+      def prune(excluding: nil)
+        Lepus.instrument :prune_processes, size: 0 do |payload|
+          arr = prunable
+          arr.delete(excluding) if excluding
+          payload[:size] = arr.size
+
+          arr.each(&:prune)
+        end
+      end
+
+      def prunable
+        ProcessRegistry.instance.all.select do |process|
+          process.last_heartbeat_at && process.last_heartbeat_at < Time.now - Lepus.config.process_alive_threshold
+        end
+      end
+    end
+
+    attr_reader :attributes
+
+    def initialize(**attributes)
+      @attributes = attributes
+      @attributes[:id] ||= SecureRandom.uuid
+    end
+
+    ATTRIBUTES.each do |attribute|
+      define_method(attribute) { attributes[attribute] }
+    end
+
+    def last_heartbeat_at
+      attributes[:last_heartbeat_at]
+    end
+
+    def rss_memory
+      MEMORY_GRABBER.call(pid)
+    end
+
+    def heartbeat
+      now = Time.now
+      Lepus.instrument :heartbeat_process, process: self, rss_memory: 0, last_heartbeat_at: now do |payload|
+        ProcessRegistry.instance.find(id) # ensure process is still registered
+
+        update_attributes(last_heartbeat_at: now)
+        payload[:rss_memory] = rss_memory
+      rescue Exception => error # rubocop:disable Lint/RescueException
+        payload[:error] = error
+        raise
+      end
+    end
+
+    def update_attributes(new_attributes)
+      @attributes = @attributes.merge(new_attributes)
+    end
+
+    def destroy!
+      Lepus.instrument :destroy_process, process: self do |payload|
+        ProcessRegistry.instance.delete(self)
+      rescue Exception => error # rubocop:disable Lint/RescueException
+        payload[:error] = error
+        raise
+      end
+    end
+
+    def deregister(pruned: false)
+      Lepus.instrument :deregister_process, process: self, pruned: pruned do |payload|
+        destroy!
+
+        unless supervised? || pruned
+          supervisees.each(&:deregister)
+        end
+      rescue Exception => error # rubocop:disable Lint/RescueException
+        payload[:error] = error
+        raise
+      end
+    end
+
+    def prune
+      deregister(pruned: true)
+    end
+
+    def supervised?
+      !attributes[:supervisor_id].nil?
+    end
+
+    def eql?(other)
+      other.is_a?(self.class) && other.id == id && other.pid == pid
+    end
+    alias_method :==, :eql?
+
+    private
+
+    def supervisees
+      ProcessRegistry.instance.all.select { |process| process.supervisor_id == id }
+    end
+  end
+end

data/lib/lepus/process_registry.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module Lepus
+  class ProcessRegistry
+    include Singleton
+
+    def initialize
+      @processes = ::Concurrent::Hash.new
+    end
+
+    def add(process)
+      @processes[process.id] = process
+    end
+
+    def delete(process)
+      @processes.delete(process.id)
+    end
+
+    def find(id)
+      @processes[id] || raise(Lepus::Process::NotFoundError.new(id))
+    end
+
+    def exists?(id)
+      @processes.key?(id)
+    end
+
+    def all
+      @processes.values
+    end
+
+    def clear
+      @processes.clear
+    end
+  end
+end

data/lib/lepus/processes/base.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Processes
+    class Base
+      include Callbacks
+      include AppExecutor
+      include Registrable
+      include Interruptible
+      include Procline
+
+      attr_reader :name
+
+      def initialize(*)
+        @name = generate_name
+        @stopped = false
+      end
+
+      def kind
+        self.class.name.split("::").last
+      end
+
+      def hostname
+        @hostname ||= Socket.gethostname.force_encoding(Encoding::UTF_8)
+      end
+
+      def pid
+        @pid ||= ::Process.pid
+      end
+
+      def metadata
+        {}
+      end
+
+      def stop
+        @stopped = true
+      end
+
+      private
+
+      def generate_name
+        [kind.downcase, SecureRandom.hex(10)].join("-")
+      end
+
+      def stopped?
+        @stopped
+      end
+    end
+  end
+end

data/lib/lepus/processes/callbacks.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Lepus::Processes
+  module Callbacks
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.send :include, InstanceMethods
+    end
+
+    module InstanceMethods
+      def run_process_callbacks(name)
+        self.class.send(:"before_#{name}_callbacks").each do |method|
+          send(method)
+        end
+
+        result = yield if block_given?
+
+        self.class.send(:"after_#{name}_callbacks").each do |method|
+          send(method)
+        end
+
+        result
+      end
+    end
+
+    module ClassMethods
+      def inherited(base)
+        base.instance_variable_set(:@before_boot_callbacks, before_boot_callbacks.dup)
+        base.instance_variable_set(:@after_boot_callbacks, after_boot_callbacks.dup)
+        base.instance_variable_set(:@before_shutdown_callbacks, before_shutdown_callbacks.dup)
+        base.instance_variable_set(:@after_shutdown_callbacks, after_shutdown_callbacks.dup)
+        super
+      end
+
+      def before_boot(*methods)
+        @before_boot_callbacks ||= []
+        @before_boot_callbacks.concat methods
+      end
+
+      def after_boot(*methods)
+        @after_boot_callbacks ||= []
+        @after_boot_callbacks.concat methods
+      end
+
+      def before_shutdown(*methods)
+        @before_shutdown_callbacks ||= []
+        @before_shutdown_callbacks.concat methods
+      end
+
+      def after_shutdown(*methods)
+        @after_shutdown_callbacks ||= []
+        @after_shutdown_callbacks.concat methods
+      end
+
+      def before_boot_callbacks
+        @before_boot_callbacks || []
+      end
+
+      def after_boot_callbacks
+        @after_boot_callbacks || []
+      end
+
+      def before_shutdown_callbacks
+        @before_shutdown_callbacks || []
+      end
+
+      def after_shutdown_callbacks
+        @after_shutdown_callbacks || []
+      end
+    end
+  end
+end

data/lib/lepus/processes/consumer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Lepus::Processes
+  class Consumer < Base
+    include Runnable
+
+    attr_reader :consumer_class
+
+    def initialize(class_name:, **options)
+      @consumer_class = class_name
+      @consumer_class = Lepus::Primitive::String.new(@consumer_class).constantize if @consumer_class.is_a?(String)
+
+      super(**options)
+    end
+
+    def metadata
+      super.merge(consumer_class: consumer_class.to_s)
+    end
+
+    def before_fork
+      return unless @consumer_class.respond_to?(:before_fork, true)
+
+      @consumer_class.send(:before_fork)
+    end
+
+    def after_fork
+      return unless @consumer_class.respond_to?(:after_fork, true)
+
+      @consumer_class.send(:after_fork)
+    end
+
+    private
+
+    SLEEP_INTERVAL = 5
+
+    def run
+      wrap_in_app_executor do
+        setup_consumer! # initialize bunny consumer within the #run method to ensure the process is running in the correct thread
+      end
+
+      loop do
+        break if shutting_down?
+
+        wrap_in_app_executor do
+          interruptible_sleep(SLEEP_INTERVAL)
+        end
+      end
+    ensure
+      Lepus.instrument(:shutdown_process, process: self) do
+        run_process_callbacks(:shutdown) { shutdown }
+      end
+    end
+
+    def shutdown
+      @subscriptions.to_a.each(&:cancel)
+      @channel&.close
+      @bunny&.close
+
+      super
+    end
+
+    def set_procline
+      procline consumer_class.name
+    end
+
+    def setup_consumer!
+      if consumer_class.config.nil?
+        raise Lepus::InvalidConsumerConfigError, "Consumer #{consumer_class.name} has no configuration"
+      end
+
+      @bunny = Thread.current[:lepus_bunny] || Lepus.config.create_connection
+      @channel = Thread.current[:lepus_channel] || begin
+        @bunny.create_channel(nil, 1, true).tap do |channel|
+          channel.prefetch(1) # @TODO make this configurable
+          channel.on_uncaught_exception { |error|
+            handle_thread_error(error)
+          }
+        end
+      end
+
+      @exchange = @channel.exchange(*consumer_class.config.exchange_args)
+      if (args = consumer_class.config.retry_queue_args)
+        @retry_queue = @channel.queue(*args)
+      end
+      if (args = consumer_class.config.error_queue_args)
+        @error_queue = @channel.queue(*args)
+      end
+
+      @subscriptions = Array.new((_threads = 1)) do |n| # may add multiple consumers in the future
+        main_queue = @channel.queue(*consumer_class.config.consumer_queue_args)
+        consumer_class.config.binds_args.each do |opts|
+          main_queue.bind(@exchange, **opts)
+        end
+
+        consumer_instance = consumer_class.new
+        consumer_wrapper = Lepus::ConsumerWrapper.new(
+          consumer_instance,
+          main_queue.channel,
+          main_queue,
+          "#{consumer_class.name}-#{n + 1}"
+        )
+        consumer_wrapper.on_delivery do |delivery_info, metadata, payload|
+          consumer_wrapper.process_delivery(delivery_info, metadata, payload)
+        end
+        main_queue.subscribe_with(consumer_wrapper)
+      end
+    rescue Bunny::TCPConnectionFailed, Bunny::PossibleAuthenticationFailureError
+      raise Lepus::ShutdownError
+    rescue Lepus::InvalidConsumerConfigError
+      raise Lepus::ShutdownError
+    end
+  end
+end

data/lib/lepus/processes/interruptible.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Lepus::Processes
+  module Interruptible
+    def wake_up
+      interrupt
+    end
+
+    private
+
+    SELF_PIPE_BLOCK_SIZE = 11
+
+    def interrupt
+      self_pipe[:writer].write_nonblock(".")
+    rescue Errno::EAGAIN, Errno::EINTR
+      # Ignore writes that would block and retry
+      # if another signal arrived while writing
+      retry
+    end
+
+    def interruptible_sleep(time)
+      if time > 0 && self_pipe[:reader].wait_readable(time)
+        loop { self_pipe[:reader].read_nonblock(SELF_PIPE_BLOCK_SIZE) }
+      end
+    rescue Errno::EAGAIN, Errno::EINTR
+    end
+
+    # Self-pipe for signal-handling (http://cr.yp.to/docs/selfpipe.html)
+    def self_pipe
+      @self_pipe ||= create_self_pipe
+    end
+
+    def create_self_pipe
+      reader, writer = IO.pipe
+      {reader: reader, writer: writer}
+    end
+  end
+end

data/lib/lepus/processes/procline.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Lepus::Processes
+  module Procline
+    # Sets the procline ($0)
+    # [lepus-supervisor: <string>]
+    def procline(string)
+      $0 = "[lepus-#{kind.downcase}: #{string}]"
+    end
+  end
+end

data/lib/lepus/processes/registrable.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Lepus::Processes
+  module Registrable
+    def self.included(base)
+      base.send :include, InstanceMethods
+      base.class_eval do
+        after_boot :register
+        after_boot :launch_heartbeat
+
+        before_shutdown :stop_heartbeat
+        after_shutdown :deregister
+      end
+    end
+
+    module InstanceMethods
+      def process_id
+        process&.id
+      end
+
+      private
+
+      attr_accessor :process
+
+      def register
+        @process = Lepus::Process.register(
+          kind: kind,
+          name: name,
+          pid: pid,
+          hostname: hostname,
+          supervisor_id: respond_to?(:supervisor) ? supervisor&.id : nil
+        )
+      end
+
+      def deregister
+        process&.deregister
+      end
+
+      def registered?
+        !!process
+      end
+
+      def launch_heartbeat
+        @heartbeat_task = ::Concurrent::TimerTask.new(execution_interval: Lepus.config.process_heartbeat_interval) do
+          wrap_in_app_executor { heartbeat }
+        end
+
+        @heartbeat_task.add_observer do |_time, _result, error|
+          handle_thread_error(error) if error
+        end
+
+        @heartbeat_task.execute
+      end
+
+      def stop_heartbeat
+        @heartbeat_task&.shutdown
+      end
+
+      def heartbeat
+        process.heartbeat
+      rescue Process::NotFoundError
+        self.process = nil
+        wake_up
+      end
+    end
+  end
+end

data/lib/lepus/processes/runnable.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Lepus::Processes
+  module Runnable
+    include Supervised
+
+    class InquiryMode
+      def initialize(mode)
+        @mode = mode.to_sym
+      end
+
+      %i[inline async fork].each do |value|
+        define_method(:"#{value}?") { @mode == value }
+      end
+    end
+
+    def start
+      boot
+
+      if running_async?
+        @thread = create_thread { run }
+      else
+        run
+      end
+    end
+
+    def stop
+      super
+
+      wake_up
+      @thread&.join
+    end
+
+    def mode=(mode)
+      @mode = InquiryMode.new(mode)
+    end
+
+    private
+
+    DEFAULT_MODE = :async
+
+    def mode
+      @mode ||= InquiryMode.new(DEFAULT_MODE)
+    end
+
+    def boot
+      Lepus.instrument(:start_process, process: self) do
+        run_process_callbacks(:boot) do
+          if running_as_fork?
+            register_signal_handlers
+            set_procline
+          end
+        end
+      end
+    end
+
+    def shutting_down?
+      stopped? || (running_as_fork? && supervisor_went_away?) || !registered? # || finished?
+    end
+
+    def run
+      raise NotImplementedError
+    end
+
+    # @TODO Add it to the inline mode
+    # def finished?
+    #   running_inline? && all_work_completed?
+    # end
+
+    # def all_work_completed?
+    #   false
+    # end
+
+    def shutdown
+    end
+
+    def set_procline
+    end
+
+    # def running_inline?
+    #   mode.inline?
+    # end
+
+    def running_async?
+      mode.async?
+    end
+
+    def running_as_fork?
+      mode.fork?
+    end
+
+    def create_thread(&block)
+      Thread.new do
+        Thread.current.name = name
+        yield
+      rescue Exception => exception # rubocop:disable Lint/RescueException
+        handle_thread_error(exception)
+        raise
+      end
+    end
+  end
+end

data/lib/lepus/processes/supervised.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Lepus::Processes
+  module Supervised
+    def self.included(base)
+      base.send :include, InstanceMethods
+      base.class_eval do
+        attr_reader :supervisor
+      end
+    end
+
+    module InstanceMethods
+      def supervised_by(process)
+        @supervisor = process
+      end
+
+      private
+
+      def set_procline
+        procline "waiting"
+      end
+
+      def supervisor_went_away?
+        supervised? && supervisor.pid != ::Process.ppid
+      end
+
+      def supervised?
+        !!supervisor
+      end
+
+      def register_signal_handlers
+        %w[INT TERM].each do |signal|
+          trap(signal) do
+            stop
+          end
+        end
+
+        trap(:QUIT) do
+          exit!
+        end
+      end
+    end
+  end
+end

data/lib/lepus/processes.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Lepus
+  module Processes
+  end
+end