cosmonats 0.5.1 → 0.7.0

data/lib/cosmo/engine.rb

@@ -13,38 +13,137 @@ module Cosmo
       instance.run(...)
     end
 
+    # Processor classes a +type+ starts: the matching one, or all of them when it names none.
+    #
+    # @param type [String, Symbol, nil]
+    # @return [Array<Class>]
+    def self.processors_for(type)
+      type && PROCESSORS.key?(type.to_sym) ? [PROCESSORS[type.to_sym]] : PROCESSORS.values
+    end
+
     def self.instance
       @instance ||= new
     end
 
+    attr_reader :concurrency
+
     def initialize
       @concurrency = Config.fetch(:concurrency, 1)
       @pool = Utils::ThreadPool.new(@concurrency)
       @running = Concurrent::AtomicBoolean.new
+      @quiet = Concurrent::AtomicBoolean.new
+      @http_server = nil
+      @heartbeat = nil
     end
 
     def run(type, options)
-      handler = Utils::Signal.trap(:INT, :TERM)
+      handler = Utils::Signal.trap(:INT, :TERM, :TSTP, :CONT, :USR1, :TTIN)
       Logger.info "Starting processing, hit Ctrl-C to stop [concurrency=#{@concurrency}]"
 
-      processor_classes = type && PROCESSORS.key?(type.to_sym) ? [PROCESSORS[type.to_sym]] : PROCESSORS.values
-      @processors = processor_classes.map { _1.run(@pool, @running, options) }
+      @processors = self.class.processors_for(type).map { _1.run(@pool, @running, options, quiet: @quiet) }
       if @running.false?
         Logger.warn "Shutting down... (No processors are running)"
         return
       end
 
-      signal = handler.wait
+      @heartbeat = Heartbeat.new(self, options:).start
+      start_http_server
+
+      signal = handle_shutdown(handler)
       Logger.info "Shutting down... (#{signal} received)"
       shutdown
     end
 
+    def running?
+      @running.true?
+    end
+
+    # @return [String] +running+, +quiet+ (no new work fetched) or +stopping+
+    def state
+      return "stopping" unless running?
+
+      @quiet.true? ? "quiet" : "running"
+    end
+
+    # @return [Integer] threads processing messages right now, across all processors
+    def busy
+      Array(@processors).sum(&:busy)
+    end
+
+    # @return [Hash{Symbol => Array<String>}] what each running processor pulls from, keyed by processor type
+    def subscriptions
+      Array(@processors).to_h { [PROCESSORS.key(_1.class), _1.subscriptions] }
+    end
+
     def shutdown
       @running.make_false
+      @heartbeat&.beat
+      @http_server&.stop
       @pool.shutdown
       Logger.info "Pausing to allow jobs to finish..."
       @pool.wait_for_termination(Config[:timeout])
+      @heartbeat&.stop
       Logger.info "Bye!"
     end
+
+    private
+
+    # HTTPServer is autoloaded, so it's referenced only after the port is checked
+    # to keep its optional dependencies (rack, webrick) unloaded otherwise.
+    def start_http_server
+      port = Config.dig(:http, :port)
+      return unless port
+
+      host = Config.dig(:http, :host) || HTTPServer::DEFAULT_HOST
+      @http_server = HTTPServer.new(port: port, host: host).start
+    end
+
+    def handle_shutdown(handler)
+      loop do
+        signal = handler.wait
+        case signal.to_s
+        when "TSTP" then quiet
+        when "CONT" then unquiet
+        when "USR1" then drain_and_exit(handler)
+        when "TTIN" then dump_threads
+        else return signal
+        end
+      end
+    end
+
+    def quiet
+      return unless @quiet.make_true
+
+      Logger.info "Received TSTP, no new jobs will be fetched; finishing in-flight work"
+      @heartbeat&.beat
+    end
+
+    def unquiet
+      return unless @quiet.make_false
+
+      Logger.info "Received CONT, resuming normal fetching"
+      @heartbeat&.beat
+    end
+
+    def drain_and_exit(handler)
+      return unless @quiet.make_true
+
+      Logger.info "Received USR1, no new jobs will be fetched, exiting when work drains"
+      @heartbeat&.beat
+      Thread.new do
+        @pool.wait_idle
+        handler.push(:TERM)
+      end
+    end
+
+    def dump_threads
+      threads = Thread.list
+      Logger.warn "Received TTIN, dumping backtraces of #{threads.size} threads"
+      threads.each do |thread|
+        backtrace = thread.backtrace || ["<no backtrace available>"]
+        tid = (thread.object_id ^ ::Process.pid).to_s(36)
+        Logger.warn "Thread tid=#{tid} name=#{thread.name} [#{thread.status}]\n#{backtrace.join("\n")}"
+      end
+    end
   end
 end

data/lib/cosmo/heartbeat.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "socket"
+require "shellwords"
+
+module Cosmo
+  # Publishes this process's identity and live state to API::Stats::Processes, so the Web UI lists
+  # every running worker, the idle ones included.
+  class Heartbeat
+    INTERVAL = 10
+
+    # @param engine [Engine]
+    # @param options [Hash] the parsed command options
+    def initialize(engine, options: {})
+      @engine = engine
+      @options = options
+      @started_at = Time.now.to_i
+    end
+
+    # Registers the process and keeps refreshing it every INTERVAL seconds.
+    #
+    # @return [self]
+    def start
+      beat
+      @thread = Thread.new do
+        loop do
+          sleep(INTERVAL)
+          beat
+        end
+      end
+      self
+    end
+
+    # Stops beating and removes the entry right away, instead of leaving it to expire.
+    #
+    # @return [void]
+    def stop
+      @thread&.kill&.join
+      @thread = nil
+      processes.unregister(identity)
+    rescue StandardError => e
+      Logger.debug "Heartbeat unregister error: #{e.class} #{e.message}"
+    end
+
+    # Writes the current state now, e.g. right after it changes.
+    #
+    # @return [void]
+    def beat
+      processes.register(identity, info)
+    rescue StandardError => e
+      Logger.debug "Heartbeat error: #{e.class} #{e.message}"
+    end
+
+    # @return [String]
+    def identity
+      @identity ||= "#{Socket.gethostname}-#{::Process.pid}"
+    end
+
+    # @return [Hash] everything the Web UI shows about this process
+    def info
+      static_info.merge(
+        state: @engine.state,
+        busy: @engine.busy,
+        subscriptions: @engine.subscriptions,
+        rss: rss,
+        nats: nats_info,
+        beat_at: Time.now.to_i
+      )
+    end
+
+    private
+
+    def static_info
+      @static_info ||= {
+        identity: identity,
+        hostname: Socket.gethostname,
+        pid: ::Process.pid,
+        cmdline: [File.basename($PROGRAM_NAME.to_s), *CLI.instance.argv].shelljoin,
+        options: @options,
+        concurrency: @engine.concurrency,
+        timeout: Config[:timeout],
+        http_port: Config.dig(:http, :port),
+        version: VERSION,
+        ruby: "#{RUBY_ENGINE} #{RUBY_VERSION}",
+        started_at: @started_at
+      }
+    end
+
+    def nats_info
+      client = Client.instance
+      connection = client.nc
+      server = connection.server_info
+      uri = connection.connected_server
+      {
+        name: client.name,
+        ip: server[:client_ip],
+        client_id: server[:client_id],
+        server: server[:server_name],
+        server_version: server[:version],
+        url: uri && "#{uri.host}:#{uri.port}",
+        rtt: rtt(connection),
+        reconnects: connection.stats[:reconnects]
+      }
+    end
+
+    # nats-pure has no RTT call, but a flush is a PING/PONG round trip.
+    def rtt(connection)
+      started = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      connection.flush(1)
+      ((::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started) * 1000).round(2)
+    rescue StandardError
+      nil
+    end
+
+    # Resident memory in KB: from procfs on Linux, from ps elsewhere.
+    def rss
+      status = "/proc/#{::Process.pid}/status"
+      return File.read(status)[/VmRSS:\s+(\d+)/, 1].to_i if File.exist?(status)
+
+      `ps -o rss= -p #{::Process.pid}`.to_i
+    rescue StandardError
+      nil
+    end
+
+    def processes
+      API::Stats::Processes.instance
+    end
+  end
+end

data/lib/cosmo/http_server/handler.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module Cosmo
+  class HTTPServer
+    # Minimal WEBrick servlet adapting requests to a Rack app.
+    class Handler < ::WEBrick::HTTPServlet::AbstractServlet
+      def initialize(server, app)
+        super(server)
+        @app = app
+      end
+
+      def service(req, res)
+        status, headers, body = @app.call(env_for(req))
+        res.status = status
+        write_headers(res, headers)
+        body.each { res.body << _1 }
+      ensure
+        body.close if body.respond_to?(:close)
+      end
+
+      private
+
+      def env_for(req)
+        env = req.meta_vars.compact
+        env.merge!(
+          Rack::SCRIPT_NAME => "",
+          Rack::PATH_INFO => req.path,
+          Rack::QUERY_STRING => req.query_string.to_s,
+          Rack::RACK_INPUT => StringIO.new(req.body.to_s).tap(&:binmode),
+          Rack::RACK_ERRORS => $stderr,
+          Rack::RACK_URL_SCHEME => req.ssl? ? "https" : "http"
+        )
+      end
+
+      def write_headers(res, headers)
+        headers.each do |key, value|
+          key.downcase == "set-cookie" ? Array(value).each { res.cookies << _1 } : res[key] = Array(value).join(", ")
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/http_server/health.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class HTTPServer
+    # Rack middleware answering GET /health, every other request goes down the stack.
+    # Responds 200 when the engine is running and NATS is connected, 503 otherwise.
+    class Health
+      PATH = "/health"
+      HEADERS = { "content-type" => "application/json", "cache-control" => "no-store" }.freeze
+
+      def initialize(app, check: nil)
+        @app = app
+        @check = check || method(:default_check)
+      end
+
+      def call(env)
+        return @app.call(env) unless env[Rack::PATH_INFO] == PATH
+        return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
+
+        checks = @check.call
+        healthy = checks.values.all?
+        body = Utils::Json.dump({ status: healthy ? "ok" : "unavailable", checks: checks })
+        [healthy ? 200 : 503, HEADERS.dup, [body]]
+      end
+
+      private
+
+      def default_check
+        { engine: Engine.instance.running?, nats: nats_connected? }
+      end
+
+      def nats_connected?
+        Client.instance.nc.connected?
+      rescue StandardError
+        false
+      end
+    end
+  end
+end

data/lib/cosmo/http_server/ping.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class HTTPServer
+    # Rack middleware answering GET /ping with 200 and no checks, every other request goes down the stack.
+    # Useful as a liveness probe: it only proves the process is up and serving HTTP.
+    class Ping
+      PATH = "/ping"
+      HEADERS = { "content-type" => "text/plain", "cache-control" => "no-store" }.freeze
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        return @app.call(env) unless env[Rack::PATH_INFO] == PATH
+        return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
+
+        [200, HEADERS.dup, ["pong"]]
+      end
+    end
+  end
+end

data/lib/cosmo/http_server.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+begin
+  require "rack"
+  require "webrick"
+rescue LoadError => e
+  raise LoadError, "Cosmo::HTTPServer requires `rack` and `webrick` gems, add them to your Gemfile (#{e.message})"
+end
+
+require "json"
+require "cosmo/http_server/handler"
+require "cosmo/http_server/health"
+require "cosmo/http_server/ping"
+
+module Cosmo
+  # WEBrick server running the Rack app in a background thread.
+  class HTTPServer
+    DEFAULT_HOST = "0.0.0.0"
+    NOT_FOUND = [404, { "content-type" => "application/json" }, ['{"error":"not found"}']].freeze
+
+    # Routing is done by middlewares, anything unmatched falls through to 404.
+    def self.app
+      Rack::Builder.app do
+        use Health
+        use Ping
+        run ->(_env) { NOT_FOUND }
+      end
+    end
+
+    def initialize(app = self.class.app, port:, host: DEFAULT_HOST)
+      @app = app
+      @host = host
+      @port = port
+    end
+
+    def start
+      @server = ::WEBrick::HTTPServer.new(
+        BindAddress: @host,
+        Port: @port,
+        Logger: ::WEBrick::Log.new(IO::NULL),
+        AccessLog: []
+      )
+      @server.mount("/", Handler, @app)
+      @thread = Thread.new { @server.start }
+      Logger.info "HTTP server listening on #{@host}:#{port}"
+      self
+    end
+
+    def stop
+      return unless @server
+
+      @server.shutdown
+      @thread&.join
+      @server = @thread = nil
+    end
+
+    # Actual bound port, useful when started with port 0.
+    def port
+      @server ? @server.listeners.first.addr[1] : @port
+    end
+  end
+end

data/lib/cosmo/job/failure.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Job
+    # Describes why a job died, as NATS headers carried alongside the payload in the
+    # dead letter queue. The worker that raised is long gone by the time the dead jobs
+    # page is rendered, so the cause has to travel with the message itself.
+    class Failure
+      HEADER_LIMIT = 1024
+      BACKTRACE_LINES = 10
+      BACKTRACE_SEPARATOR = " | "
+
+      # @param exception [Exception]
+      # @return [Hash{String => String}] headers describing the failure
+      def self.headers(exception)
+        new(exception).headers
+      end
+
+      # @param exception [Exception]
+      def initialize(exception)
+        @exception = exception
+      end
+
+      # @return [Hash{String => String}]
+      def headers
+        headers = { "X-Error-Class" => name, "X-Error-Message" => message }
+        headers["X-Error-Backtrace"] = backtrace unless backtrace.empty?
+        headers
+      end
+
+      private
+
+      def name
+        @exception.class.name.to_s
+      end
+
+      def message
+        value = single_line(@exception.message)
+        value.empty? ? name : value
+      end
+
+      def backtrace
+        @backtrace ||= single_line(Array(@exception.backtrace).first(BACKTRACE_LINES).join(BACKTRACE_SEPARATOR))
+      end
+
+      # NATS header values are a single line, so whitespace collapses and the value is capped.
+      def single_line(value)
+        value.to_s.gsub(/\s+/, " ").strip[0, HEADER_LIMIT].to_s
+      end
+    end
+  end
+end