cosmonats 0.6.0 → 0.8.0

data/lib/cosmo/engine.rb

@@ -13,38 +13,137 @@ module Cosmo
       instance.run(...)
     end
 
+    # Processor classes a +type+ starts: the matching one, or all of them when it names none.
+    #
+    # @param type [String, Symbol, nil]
+    # @return [Array<Class>]
+    def self.processors_for(type)
+      type && PROCESSORS.key?(type.to_sym) ? [PROCESSORS[type.to_sym]] : PROCESSORS.values
+    end
+
     def self.instance
       @instance ||= new
     end
 
+    attr_reader :concurrency
+
     def initialize
       @concurrency = Config.fetch(:concurrency, 1)
       @pool = Utils::ThreadPool.new(@concurrency)
       @running = Concurrent::AtomicBoolean.new
+      @quiet = Concurrent::AtomicBoolean.new
+      @http_server = nil
+      @heartbeat = nil
     end
 
     def run(type, options)
-      handler = Utils::Signal.trap(:INT, :TERM)
+      handler = Utils::Signal.trap(:INT, :TERM, :TSTP, :CONT, :USR1, :TTIN)
       Logger.info "Starting processing, hit Ctrl-C to stop [concurrency=#{@concurrency}]"
 
-      processor_classes = type && PROCESSORS.key?(type.to_sym) ? [PROCESSORS[type.to_sym]] : PROCESSORS.values
-      @processors = processor_classes.map { _1.run(@pool, @running, options) }
+      @processors = self.class.processors_for(type).map { _1.run(@pool, @running, options, quiet: @quiet) }
       if @running.false?
         Logger.warn "Shutting down... (No processors are running)"
         return
       end
 
-      signal = handler.wait
+      @heartbeat = Heartbeat.new(self, options:).start
+      start_http_server
+
+      signal = handle_shutdown(handler)
       Logger.info "Shutting down... (#{signal} received)"
       shutdown
     end
 
+    def running?
+      @running.true?
+    end
+
+    # @return [String] +running+, +quiet+ (no new work fetched) or +stopping+
+    def state
+      return "stopping" unless running?
+
+      @quiet.true? ? "quiet" : "running"
+    end
+
+    # @return [Integer] threads processing messages right now, across all processors
+    def busy
+      Array(@processors).sum(&:busy)
+    end
+
+    # @return [Hash{Symbol => Array<String>}] what each running processor pulls from, keyed by processor type
+    def subscriptions
+      Array(@processors).to_h { [PROCESSORS.key(_1.class), _1.subscriptions] }
+    end
+
     def shutdown
       @running.make_false
+      @heartbeat&.beat
+      @http_server&.stop
       @pool.shutdown
       Logger.info "Pausing to allow jobs to finish..."
       @pool.wait_for_termination(Config[:timeout])
+      @heartbeat&.stop
       Logger.info "Bye!"
     end
+
+    private
+
+    # HTTPServer is autoloaded, so it's referenced only after the port is checked
+    # to keep its optional dependencies (rack, webrick) unloaded otherwise.
+    def start_http_server
+      port = Config.dig(:http, :port)
+      return unless port
+
+      host = Config.dig(:http, :host) || HTTPServer::DEFAULT_HOST
+      @http_server = HTTPServer.new(port: port, host: host).start
+    end
+
+    def handle_shutdown(handler)
+      loop do
+        signal = handler.wait
+        case signal.to_s
+        when "TSTP" then quiet
+        when "CONT" then unquiet
+        when "USR1" then drain_and_exit(handler)
+        when "TTIN" then dump_threads
+        else return signal
+        end
+      end
+    end
+
+    def quiet
+      return unless @quiet.make_true
+
+      Logger.info "Received TSTP, no new jobs will be fetched; finishing in-flight work"
+      @heartbeat&.beat
+    end
+
+    def unquiet
+      return unless @quiet.make_false
+
+      Logger.info "Received CONT, resuming normal fetching"
+      @heartbeat&.beat
+    end
+
+    def drain_and_exit(handler)
+      return unless @quiet.make_true
+
+      Logger.info "Received USR1, no new jobs will be fetched, exiting when work drains"
+      @heartbeat&.beat
+      Thread.new do
+        @pool.wait_idle
+        handler.push(:TERM)
+      end
+    end
+
+    def dump_threads
+      threads = Thread.list
+      Logger.warn "Received TTIN, dumping backtraces of #{threads.size} threads"
+      threads.each do |thread|
+        backtrace = thread.backtrace || ["<no backtrace available>"]
+        tid = (thread.object_id ^ ::Process.pid).to_s(36)
+        Logger.warn "Thread tid=#{tid} name=#{thread.name} [#{thread.status}]\n#{backtrace.join("\n")}"
+      end
+    end
   end
 end

data/lib/cosmo/heartbeat.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require "socket"
+require "shellwords"
+
+module Cosmo
+  # Publishes this process's identity and live state to API::Stats::Processes, so the Web UI lists
+  # every running worker, the idle ones included.
+  class Heartbeat
+    INTERVAL = 10
+
+    # @param engine [Engine]
+    # @param options [Hash] the parsed command options
+    def initialize(engine, options: {})
+      @engine = engine
+      @options = options
+      @started_at = Time.now.to_i
+    end
+
+    # Registers the process and keeps refreshing it every INTERVAL seconds.
+    #
+    # @return [self]
+    def start
+      beat
+      @thread = Thread.new do
+        loop do
+          sleep(INTERVAL)
+          beat
+          flush
+        end
+      end
+      self
+    end
+
+    # Stops beating and removes the entry right away, instead of leaving it to expire.
+    #
+    # @return [void]
+    def stop
+      @thread&.kill&.join
+      @thread = nil
+      flush
+      processes.unregister(identity)
+    rescue StandardError => e
+      Logger.debug "Heartbeat unregister error: #{e.class} #{e.message}"
+    end
+
+    # Writes the current state now, e.g. right after it changes.
+    #
+    # @return [void]
+    def beat
+      processes.register(identity, info)
+    rescue StandardError => e
+      Logger.debug "Heartbeat error: #{e.class} #{e.message}"
+    end
+
+    # Writes the job metrics recorded since the last flush.
+    #
+    # @return [void]
+    def flush
+      API::Stats::Metrics.instance.flush if Config.metrics.enabled
+    rescue StandardError => e
+      Logger.debug "Metrics flush error: #{e.class} #{e.message}"
+    end
+
+    # @return [String]
+    def identity
+      @identity ||= "#{Socket.gethostname}-#{::Process.pid}"
+    end
+
+    # @return [Hash] everything the Web UI shows about this process
+    def info
+      static_info.merge(
+        state: @engine.state,
+        busy: @engine.busy,
+        subscriptions: @engine.subscriptions,
+        rss: rss,
+        nats: nats_info,
+        beat_at: Time.now.to_i
+      )
+    end
+
+    private
+
+    def static_info
+      @static_info ||= {
+        identity: identity,
+        hostname: Socket.gethostname,
+        pid: ::Process.pid,
+        cmdline: [File.basename($PROGRAM_NAME.to_s), *CLI.instance.argv].shelljoin,
+        options: @options,
+        concurrency: @engine.concurrency,
+        timeout: Config[:timeout],
+        http_port: Config.dig(:http, :port),
+        version: VERSION,
+        ruby: "#{RUBY_ENGINE} #{RUBY_VERSION}",
+        started_at: @started_at
+      }
+    end
+
+    def nats_info
+      client = Client.instance
+      connection = client.nc
+      server = connection.server_info
+      uri = connection.connected_server
+      {
+        name: client.name,
+        ip: server[:client_ip],
+        client_id: server[:client_id],
+        server: server[:server_name],
+        server_version: server[:version],
+        url: uri && "#{uri.host}:#{uri.port}",
+        rtt: rtt(connection),
+        reconnects: connection.stats[:reconnects]
+      }
+    end
+
+    # nats-pure has no RTT call, but a flush is a PING/PONG round trip.
+    def rtt(connection)
+      started = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      connection.flush(1)
+      ((::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started) * 1000).round(2)
+    rescue StandardError
+      nil
+    end
+
+    # Resident memory in KB: from procfs on Linux, from ps elsewhere.
+    def rss
+      status = "/proc/#{::Process.pid}/status"
+      return File.read(status)[/VmRSS:\s+(\d+)/, 1].to_i if File.exist?(status)
+
+      `ps -o rss= -p #{::Process.pid}`.to_i
+    rescue StandardError
+      nil
+    end
+
+    def processes
+      API::Stats::Processes.instance
+    end
+  end
+end

data/lib/cosmo/http_server/handler.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module Cosmo
+  class HTTPServer
+    # Minimal WEBrick servlet adapting requests to a Rack app.
+    class Handler < ::WEBrick::HTTPServlet::AbstractServlet
+      def initialize(server, app)
+        super(server)
+        @app = app
+      end
+
+      def service(req, res)
+        status, headers, body = @app.call(env_for(req))
+        res.status = status
+        write_headers(res, headers)
+        body.each { res.body << _1 }
+      ensure
+        body.close if body.respond_to?(:close)
+      end
+
+      private
+
+      def env_for(req)
+        env = req.meta_vars.compact
+        env.merge!(
+          Rack::SCRIPT_NAME => "",
+          Rack::PATH_INFO => req.path,
+          Rack::QUERY_STRING => req.query_string.to_s,
+          Rack::RACK_INPUT => StringIO.new(req.body.to_s).tap(&:binmode),
+          Rack::RACK_ERRORS => $stderr,
+          Rack::RACK_URL_SCHEME => req.ssl? ? "https" : "http"
+        )
+      end
+
+      def write_headers(res, headers)
+        headers.each do |key, value|
+          key.downcase == "set-cookie" ? Array(value).each { res.cookies << _1 } : res[key] = Array(value).join(", ")
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/http_server/health.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class HTTPServer
+    # Rack middleware answering GET /health, every other request goes down the stack.
+    # Responds 200 when the engine is running and NATS is connected, 503 otherwise.
+    class Health
+      PATH = "/health"
+      HEADERS = { "content-type" => "application/json", "cache-control" => "no-store" }.freeze
+
+      def initialize(app, check: nil)
+        @app = app
+        @check = check || method(:default_check)
+      end
+
+      def call(env)
+        return @app.call(env) unless env[Rack::PATH_INFO] == PATH
+        return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
+
+        checks = @check.call
+        healthy = checks.values.all?
+        body = Utils::Json.dump({ status: healthy ? "ok" : "unavailable", checks: checks })
+        [healthy ? 200 : 503, HEADERS.dup, [body]]
+      end
+
+      private
+
+      def default_check
+        { engine: Engine.instance.running?, nats: nats_connected? }
+      end
+
+      def nats_connected?
+        Client.instance.nc.connected?
+      rescue StandardError
+        false
+      end
+    end
+  end
+end

data/lib/cosmo/http_server/ping.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class HTTPServer
+    # Rack middleware answering GET /ping with 200 and no checks, every other request goes down the stack.
+    # Useful as a liveness probe: it only proves the process is up and serving HTTP.
+    class Ping
+      PATH = "/ping"
+      HEADERS = { "content-type" => "text/plain", "cache-control" => "no-store" }.freeze
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        return @app.call(env) unless env[Rack::PATH_INFO] == PATH
+        return [405, HEADERS.merge("allow" => "GET, HEAD"), []] unless %w[GET HEAD].include?(env[Rack::REQUEST_METHOD])
+
+        [200, HEADERS.dup, ["pong"]]
+      end
+    end
+  end
+end

data/lib/cosmo/http_server.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+begin
+  require "rack"
+  require "webrick"
+rescue LoadError => e
+  raise LoadError, "Cosmo::HTTPServer requires `rack` and `webrick` gems, add them to your Gemfile (#{e.message})"
+end
+
+require "json"
+require "cosmo/http_server/handler"
+require "cosmo/http_server/health"
+require "cosmo/http_server/ping"
+
+module Cosmo
+  # WEBrick server running the Rack app in a background thread.
+  class HTTPServer
+    DEFAULT_HOST = "0.0.0.0"
+    NOT_FOUND = [404, { "content-type" => "application/json" }, ['{"error":"not found"}']].freeze
+
+    # Routing is done by middlewares, anything unmatched falls through to 404.
+    def self.app
+      Rack::Builder.app do
+        use Health
+        use Ping
+        run ->(_env) { NOT_FOUND }
+      end
+    end
+
+    def initialize(app = self.class.app, port:, host: DEFAULT_HOST)
+      @app = app
+      @host = host
+      @port = port
+    end
+
+    def start
+      @server = ::WEBrick::HTTPServer.new(
+        BindAddress: @host,
+        Port: @port,
+        Logger: ::WEBrick::Log.new(IO::NULL),
+        AccessLog: []
+      )
+      @server.mount("/", Handler, @app)
+      @thread = Thread.new { @server.start }
+      Logger.info "HTTP server listening on #{@host}:#{port}"
+      self
+    end
+
+    def stop
+      return unless @server
+
+      @server.shutdown
+      @thread&.join
+      @server = @thread = nil
+    end
+
+    # Actual bound port, useful when started with port 0.
+    def port
+      @server ? @server.listeners.first.addr[1] : @port
+    end
+  end
+end

data/lib/cosmo/job/data.rb

@@ -36,8 +36,19 @@ module Cosmo
         @at ? :scheduled : @options[:stream]
       end
 
-      def subject(target: false)
-        ["jobs", stream(target:).to_s, Utils::String.underscore(@class_name)]
+      # @return [String] the subject the job is published to: the custom +subject:+ option, or the one derived
+      #   from its stream and class
+      def subject
+        @subject || subject_for(stream)
+      end
+
+      # @return [Hash{String => String, Integer}] dedup id, plus where to dispatch a delayed job once it is due
+      def headers
+        headers = { "Nats-Msg-Id" => jid }
+        return headers unless @at
+
+        target = stream(target: true)
+        headers.merge("X-Execute-At" => @at.to_i, "X-Stream" => target, "X-Subject" => subject_for(target))
       end
 
       def as_json
@@ -49,18 +60,12 @@ module Cosmo
         Utils::Json.dump(as_json)
       end
 
-      def to_args
-        headers = { "Nats-Msg-Id" => jid }
-        if @at
-          headers.merge!("X-Execute-At" => @at.to_i,
-                         "X-Stream" => stream(target: true),
-                         "X-Subject" => subject(target: true).join("."))
-        end
-        [@subject || subject.join("."), to_json, { stream: stream, header: headers }]
-      end
-
       private
 
+      def subject_for(stream)
+        "jobs.#{stream}.#{Utils::String.underscore(@class_name)}"
+      end
+
       def validate!
         raise ArgumentError, "stream is not provided" unless @options[:stream]
       end

data/lib/cosmo/job/enqueuer.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Job
+    # The one path a job takes onto its stream, shared by {Job::ClassMethods#perform} and the ActiveJob adapter.
+    module Enqueuer
+      module_function
+
+      # Publishes the job through {Config#client_middleware}.
+      #
+      # @param class_name [String]
+      # @param args [Array]
+      # @param options [Hash] job options, see {Data}
+      # @param batch [Batch, nil] the batch the job joins; it is reserved a pending slot before the publish
+      #   (see Batch#jobs) and released again when the publish fails or a middleware stops it, so the batch never
+      #   waits on a lost job
+      # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
+      # @raise [StreamNotFoundError] when the target stream does not exist
+      # @raise [SchedulingDisabledError] for a delayed job while scheduling is turned off
+      def enqueue(class_name, args, options, batch: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+        options = options.merge(batch_id: batch.bid) if batch
+        data = Data.new(class_name, args, options)
+        raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
+
+        batch&.register_job!
+        begin
+          published = pipe(class_name, data)
+        rescue StandardError
+          batch&.rollback_job!
+          raise
+        end
+        batch&.rollback_job! unless published
+        data.jid if published
+      end
+
+      def pipe(class_name, data)
+        payload = data.as_json
+        published = false
+        Config.client_middleware.invoke(class_name, payload, data.stream(target: true)) do
+          publish(data, payload)
+          published = true
+        end
+        published
+      end
+
+      def publish(data, payload)
+        Client.instance.publish(data.subject, Utils::Json.dump(payload), stream: data.stream, header: data.headers)
+      rescue NATS::JetStream::Error::NoStreamResponse
+        raise StreamNotFoundError, data.stream.to_s
+      end
+    end
+  end
+end

data/lib/cosmo/job/failure.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Job
+    # Describes why a job died, as NATS headers carried alongside the payload in the
+    # dead letter queue. The worker that raised is long gone by the time the dead jobs
+    # page is rendered, so the cause has to travel with the message itself.
+    class Failure
+      HEADER_LIMIT = 1024
+      BACKTRACE_LINES = 10
+      BACKTRACE_SEPARATOR = " | "
+
+      # @param exception [Exception]
+      # @return [Hash{String => String}] headers describing the failure
+      def self.headers(exception)
+        new(exception).headers
+      end
+
+      # @param exception [Exception]
+      def initialize(exception)
+        @exception = exception
+      end
+
+      # @return [Hash{String => String}]
+      def headers
+        headers = { "X-Error-Class" => name, "X-Error-Message" => message }
+        headers["X-Error-Backtrace"] = backtrace unless backtrace.empty?
+        headers
+      end
+
+      private
+
+      def name
+        @exception.class.name.to_s
+      end
+
+      def message
+        value = single_line(@exception.message)
+        value.empty? ? name : value
+      end
+
+      def backtrace
+        @backtrace ||= single_line(Array(@exception.backtrace).first(BACKTRACE_LINES).join(BACKTRACE_SEPARATOR))
+      end
+
+      # NATS header values are a single line, so whitespace collapses and the value is capped.
+      def single_line(value)
+        value.to_s.gsub(/\s+/, " ").strip[0, HEADER_LIMIT].to_s
+      end
+    end
+  end
+end

data/lib/cosmo/job/limit.rb

@@ -13,14 +13,14 @@ module Cosmo
     # expires it automatically via the per-message Nats-TTL header. Both
     # paths leave the slot equally empty -- no tombstone, no delete marker.
     class Limit
-      BUCKET = "cosmo_jobs_limits"
+      BUCKET = "_cosmolimits"
 
       def self.instance
         @instance ||= new
       end
 
       def initialize
-        @kv = API::KV.new(BUCKET, allow_msg_ttl: true)
+        @kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.replicas)
       end
 
       # Try to acquire one of the numbered slots for +key+.