cosmonats 0.7.0 → 0.9.0

data/lib/cosmo/config.rb

@@ -2,31 +2,68 @@
 
 require "yaml"
 require "forwardable"
+require "cosmo/config/settings"
 
 module Cosmo
   class Config < ::Hash
     NANO = 1_000_000_000
     DEFAULT_PATH = "config/cosmo.yml"
+    DEFAULTS_FILE = File.expand_path(DEFAULT_PATH, __dir__)
+    SERVICE_STREAMS = %i[scheduled dead].freeze
 
     class << self
       extend Forwardable
 
-      delegate %i[[] fetch dig to_h set load] => :instance
+      delegate %i[[] fetch dig load server_middleware client_middleware error_handlers replicas scheduled dead batches metrics] => :instance
     end
 
     def self.to_ns(seconds)
       (seconds.to_f * NANO).to_i
     end
 
+    def self.read(path)
+      Utils::Hash.symbolize_keys!(YAML.load_file(path, aliases: true) || {})
+    end
+
     def self.parse_file(path)
-      YAML.load_file(path, aliases: true).tap { normalize!(_1) }
+      read(path).tap { normalize!(_1) }
+    end
+
+    # The built-in {DEFAULTS_FILE} with +user+ deep-merged over it. Job streams are the one list that is not merged:
+    # a user who lists any in +setup.jobs+ gets exactly those, and the built-in +default+ stream and consumer go away.
+    # Without +setup.jobs+, +consumers.jobs.default+ still tunes the built-in consumer.
+    #
+    # @param user [Hash] a parsed config file, not yet normalized
+    # @return [Hash] the normalized effective config
+    # @raise [ConfigError] when +user+ configures something Cosmo.configure owns
+    def self.build(user)
+      validate!(user)
+      defaults = read(DEFAULTS_FILE)
+      if user.dig(:setup, :jobs)
+        defaults[:setup].delete(:jobs)
+        defaults[:consumers].delete(:jobs)
+      end
+      Utils::Hash.deep_merge(defaults, user).tap { normalize!(_1) }
+    end
+
+    def self.validate!(config)
+      %i[setup consumers].each do |section|
+        jobs = config.dig(section, :jobs)
+        name = SERVICE_STREAMS.find { jobs.is_a?(::Hash) && jobs.key?(_1) }
+        next unless name
+
+        raise ConfigError, "`#{section}.jobs.#{name}` is a Cosmo service stream: remove it from cosmo.yml " \
+                           "and tune it with Cosmo.configure { |config| config.#{name} }"
+      end
+      return unless config.key?(:batch_expiry)
+
+      raise ConfigError, "`batch_expiry` moved out of cosmo.yml: Cosmo.configure { |config| config.batches.expiry = ... }"
     end
 
     def self.normalize!(config) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       Utils::Hash.symbolize_keys!(config)
 
       config[:timeout] = Utils::Duration.parse(config[:timeout]) if config[:timeout]
-      config[:batch_expiry] = Utils::Duration.parse(config[:batch_expiry]) if config[:batch_expiry]
 
       config[:consumers]&.each_key do |name|
         config[:consumers][name].each do |stream_name, c|
@@ -46,17 +83,6 @@ module Cosmo
           c[:max_age] = to_ns(Utils::Duration.parse(c[:max_age])) if c[:max_age]
           c[:duplicate_window] = to_ns(Utils::Duration.parse(c[:duplicate_window])) if c[:duplicate_window]
           c[:subjects] = c[:subjects].map { |s| format(s, name: name) } if c[:subjects]
-
-          # The scheduled stream carries every cron template, so it alone enables NATS message
-          # scheduling. A schedule can only target a subject its own stream covers, and NATS
-          # rejects `discard: new` on any stream with scheduling enabled - keeping both here
-          # leaves the streams that actually run jobs free to choose their own discard policy.
-          next unless type == :jobs && name.to_s == Job::StreamFilter::SCHEDULED.to_s
-
-          c[:allow_msg_schedules] = true
-          cron_subject = "#{API::Cron::Entry::SUBJECT_PREFIX}.>"
-          c[:subjects] = Array(c[:subjects])
-          c[:subjects] << cron_subject unless c[:subjects].include?(cron_subject)
         end
       end
     end
@@ -84,14 +110,113 @@ module Cosmo
       @internal ||= {}
     end
 
-    def set(...)
-      Utils::Hash.set(self, ...)
+    # Replaces the contents with {.build} of the file at +path+ (or of no file), with +overrides+ merged over it, so
+    # command-line flags win over the file.
+    #
+    # @param path [String, nil]
+    # @param overrides [Hash, nil] e.g. +{ concurrency: 5 }+
+    def load(path = nil, overrides: nil)
+      user = path ? self.class.read(path) : {}
+      replace(self.class.build(Utils::Hash.deep_merge(user, Hash(overrides))))
+    end
+
+    attr_writer :replicas
+
+    # @return [Integer] replicas for every service stream and bucket, e.g. 3 on a NATS cluster
+    def replicas
+      @replicas || 1
+    end
+
+    # @return [Scheduled]
+    def scheduled
+      @scheduled ||= Scheduled.new(enabled: true)
     end
 
-    def load(path = nil)
-      return unless path
+    # @return [Dead]
+    def dead
+      @dead ||= Dead.new(enabled: true, max_age: 7 * 86_400, max_msgs: 10_000, max_bytes: -1)
+    end
+
+    # @return [Batches]
+    def batches
+      @batches ||= Batches.new(expiry: 3 * 86_400)
+    end
+
+    # @return [Metrics]
+    def metrics
+      @metrics ||= Metrics.new(enabled: true, retention: "30d")
+    end
+
+    # Callables given every error Cosmo rescues: failed jobs and stream batches, fetch and scheduler errors, rejected
+    # messages. Each is called with +(error, context)+, +context+ being a Hash with a +:source+ (+:job+, +:stream+,
+    # +:fetch+, +:scheduler+, +:reject+, +:retry_in+, +:limit+) and what is known there, e.g. a job's payload.
+    #
+    #   Cosmo.configure do |config|
+    #     config.error_handlers << ->(error, context) { Honeybadger.notify(error, context: context) }
+    #   end
+    #
+    # @return [Array<#call>]
+    def error_handlers
+      @error_handlers ||= []
+    end
+
+    # @return [::Logger] the logger Cosmo writes to, see {Logger.instance}
+    def logger
+      Logger.instance
+    end
+
+    # Replaces Cosmo's stdout logger, e.g. with +Rails.logger+. Give it +Cosmo::Logger::SimpleFormatter.new+ to keep
+    # Cosmo's +jid+/+elapsed+ context in each line.
+    #
+    # @param logger [::Logger]
+    def logger=(logger)
+      Logger.instance = logger
+    end
+
+    # Ignored when +COSMO_LOG_LEVEL+ is set, which always wins.
+    #
+    # @param level [Symbol, String, Integer] +:trace+, +:debug+, +:info+, +:warn+, +:error+, or +:fatal+
+    def log_level=(level)
+      Logger.level = level
+    end
+
+    # The chain every job execution runs through, starting with the built-in {Middleware::Limit}, {Middleware::Busy},
+    # {Middleware::Totals}, and {Middleware::Metrics}. It lives outside the loaded YAML, so {#load} keeps it. Register middleware
+    # at boot, before workers start:
+    #
+    #   Cosmo.configure do |config|
+    #     config.server_middleware do |chain|
+    #       chain.add MyMiddleware
+    #     end
+    #   end
+    #
+    # @yieldparam chain [Middleware::Chain]
+    # @return [Middleware::Chain]
+    def server_middleware
+      @server_middleware ||= Middleware::Chain.new do |chain|
+        chain.add Middleware::Limit
+        chain.add Middleware::Busy
+        chain.add Middleware::Totals
+        chain.add Middleware::Metrics
+      end
+      yield @server_middleware if block_given?
+      @server_middleware
+    end
 
-      replace(self.class.parse_file(path))
+    # The chain every job enqueue runs through, empty by default. A middleware is called with
+    # +(job_class_name, payload, stream)+ around the publish: it may change +payload+ (the Hash that gets published),
+    # or not yield to stop the publish, so +perform_async+ returns +nil+. +stream+ is the stream the job runs on.
+    #
+    #   Cosmo.configure do |config|
+    #     config.client_middleware { |chain| chain.add RequestIdMiddleware }
+    #   end
+    #
+    # @yieldparam chain [Middleware::Chain]
+    # @return [Middleware::Chain]
+    def client_middleware
+      @client_middleware ||= Middleware::Chain.new
+      yield @client_middleware if block_given?
+      @client_middleware
     end
   end
 end

data/lib/cosmo/heartbeat.rb

@@ -26,6 +26,7 @@ module Cosmo
         loop do
           sleep(INTERVAL)
           beat
+          flush
         end
       end
       self
@@ -37,6 +38,7 @@ module Cosmo
     def stop
       @thread&.kill&.join
       @thread = nil
+      flush
       processes.unregister(identity)
     rescue StandardError => e
       Logger.debug "Heartbeat unregister error: #{e.class} #{e.message}"
@@ -51,6 +53,15 @@ module Cosmo
       Logger.debug "Heartbeat error: #{e.class} #{e.message}"
     end
 
+    # Writes the job metrics recorded since the last flush.
+    #
+    # @return [void]
+    def flush
+      API::Stats::Metrics.instance.flush if Config.metrics.enabled
+    rescue StandardError => e
+      Logger.debug "Metrics flush error: #{e.class} #{e.message}"
+    end
+
     # @return [String]
     def identity
       @identity ||= "#{Socket.gethostname}-#{::Process.pid}"

data/lib/cosmo/job/data.rb

@@ -36,8 +36,19 @@ module Cosmo
         @at ? :scheduled : @options[:stream]
       end
 
-      def subject(target: false)
-        ["jobs", stream(target:).to_s, Utils::String.underscore(@class_name)]
+      # @return [String] the subject the job is published to: the custom +subject:+ option, or the one derived
+      #   from its stream and class
+      def subject
+        @subject || subject_for(stream)
+      end
+
+      # @return [Hash{String => String, Integer}] dedup id, plus where to dispatch a delayed job once it is due
+      def headers
+        headers = { NATS::JetStream::Header::MSG_ID => jid }
+        return headers unless @at
+
+        target = stream(target: true)
+        headers.merge("X-Execute-At" => @at.to_i, "X-Stream" => target, "X-Subject" => subject_for(target))
       end
 
       def as_json
@@ -49,18 +60,12 @@ module Cosmo
         Utils::Json.dump(as_json)
       end
 
-      def to_args
-        headers = { "Nats-Msg-Id" => jid }
-        if @at
-          headers.merge!("X-Execute-At" => @at.to_i,
-                         "X-Stream" => stream(target: true),
-                         "X-Subject" => subject(target: true).join("."))
-        end
-        [@subject || subject.join("."), to_json, { stream: stream, header: headers }]
-      end
-
       private
 
+      def subject_for(stream)
+        "jobs.#{stream}.#{Utils::String.underscore(@class_name)}"
+      end
+
       def validate!
         raise ArgumentError, "stream is not provided" unless @options[:stream]
       end

data/lib/cosmo/job/enqueuer.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Job
+    # The one path a job takes onto its stream, shared by {Job::ClassMethods#perform} and the ActiveJob adapter.
+    module Enqueuer
+      module_function
+
+      # Publishes the job through {Config#client_middleware}.
+      #
+      # @param class_name [String]
+      # @param args [Array]
+      # @param options [Hash] job options, see {Data}
+      # @param batch [Batch, nil] the batch the job joins; it is reserved a pending slot before the publish
+      #   (see Batch#jobs) and released again when the publish fails or a middleware stops it, so the batch never
+      #   waits on a lost job
+      # @return [String, nil] the job's jid, or nil when a client middleware stopped the publish
+      # @raise [StreamNotFoundError] when the target stream does not exist
+      # @raise [SchedulingDisabledError] for a delayed job while scheduling is turned off
+      def enqueue(class_name, args, options, batch: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+        options = options.merge(batch_id: batch.bid) if batch
+        data = Data.new(class_name, args, options)
+        raise SchedulingDisabledError if data.stream.to_s == Services::SCHEDULED && !Config.scheduled.enabled
+
+        batch&.register_job!
+        begin
+          published = pipe(class_name, data)
+        rescue StandardError
+          batch&.rollback_job!
+          raise
+        end
+        batch&.rollback_job! unless published
+        data.jid if published
+      end
+
+      def pipe(class_name, data)
+        payload = data.as_json
+        published = false
+        Config.client_middleware.invoke(class_name, payload, data.stream(target: true)) do
+          publish(data, payload)
+          published = true
+        end
+        published
+      end
+
+      def publish(data, payload)
+        Client.instance.publish(data.subject, Utils::Json.dump(payload), stream: data.stream, header: data.headers)
+      rescue NATS::JetStream::Error::NoStreamResponse
+        raise StreamNotFoundError, data.stream.to_s
+      end
+    end
+  end
+end

data/lib/cosmo/job/limit.rb

@@ -13,14 +13,14 @@ module Cosmo
     # expires it automatically via the per-message Nats-TTL header. Both
     # paths leave the slot equally empty -- no tombstone, no delete marker.
     class Limit
-      BUCKET = "cosmo_jobs_limits"
+      BUCKET = "_cosmolimits"
 
       def self.instance
         @instance ||= new
       end
 
       def initialize
-        @kv = API::KV.new(BUCKET, allow_msg_ttl: true)
+        @kv = API::KV.new(BUCKET, allow_msg_ttl: true, replicas: Config.replicas)
       end
 
       # Try to acquire one of the numbered slots for +key+.

data/lib/cosmo/job/processor.rb

@@ -8,8 +8,8 @@ module Cosmo
       # Headers the scheduler consumes rather than forwards. The +X-+ ones address the dispatch
       # itself, and NATS sets +Nats-Scheduler+/+Nats-Schedule-Next+ on a message it fires and
       # rejects a publish that carries them back.
-      DISPATCH_HEADERS = %w[X-Stream X-Subject X-Execute-At Nats-Expected-Stream
-                            Nats-Scheduler Nats-Schedule-Next].freeze
+      DISPATCH_HEADERS = ["X-Stream", "X-Subject", "X-Execute-At", NATS::JetStream::Header::EXPECTED_STREAM,
+                          NATS::JetStream::Header::SCHEDULER, NATS::JetStream::Header::SCHEDULE_NEXT].freeze
 
       # @raise [UnknownJobStreamError] when --streams/--stream or COSMO_JOBS_STREAMS names an unconfigured stream
       def self.validate_options!(options)
@@ -32,6 +32,7 @@ module Cosmo
         API::Stats::Busy.instance
         API::Stats::Totals.instance
         Limit.instance
+        Config.server_middleware
 
         jobs_config = Config.dig(:consumers, :jobs)
         jobs_config&.each do |stream_name, config|
@@ -53,7 +54,7 @@ module Cosmo
       end
 
       def scheduled_config
-        Config.dig(:consumers, :jobs, StreamFilter::SCHEDULED)
+        Services.scheduled_consumer
       end
 
       def schedule_loop # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
@@ -70,8 +71,7 @@ module Cosmo
           messages&.each do |message|
             headers = message.header.except(*DISPATCH_HEADERS)
             stream, subject, execute_at = message.header.values_at("X-Stream", "X-Subject", "X-Execute-At")
-            headers["Nats-Expected-Stream"] = stream
-            scheduler = message.header["Nats-Scheduler"]
+            scheduler = message.header[NATS::JetStream::Header::SCHEDULER]
             if scheduler
               headers["X-Scheduled-By"] = scheduler
               headers["X-Enqueued-At"] = message.metadata.timestamp.to_f.to_s
@@ -79,7 +79,7 @@ module Cosmo
             execute_at = execute_at.to_i
 
             if now >= execute_at
-              client.publish(subject, message.data, header: headers)
+              client.publish(subject, message.data, header: headers, stream:)
               message.ack
             else
               message.nak(delay: Config.to_ns(execute_at - now))
@@ -89,6 +89,7 @@ module Cosmo
             # to escape #each and kill this thread — schedule_loop only runs once per processor,
             # so an unhandled exception would silently stop all future scheduled-job dispatch.
             Logger.error e
+            Cosmo.handle_error(e, { source: :scheduler, subject: message.subject })
             message.nak rescue nil
           end
 
@@ -96,7 +97,7 @@ module Cosmo
         end
       end
 
-      def process(messages, _) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def process(messages, _) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         message = messages.first
         Logger.debug "received messages #{messages.inspect}"
         data = Utils::Json.parse(message.data)
@@ -109,40 +110,36 @@ module Cosmo
           return
         end
 
-        if worker_class.limits_concurrency?
-          slot = acquire_concurrency_slot(worker_class, message, data)
-          return if slot == false
-        end
-
-        duration = worker_class.default_options[:limit]&.dig(:duration)&.to_i
-
-        with_stats(message) do
+        begin
           sw = stopwatch
           Logger.with(jid: data[:jid])
           Logger.info "start"
 
           instance = build_worker(worker_class, data, message)
-          perform_job(instance, data: data, message: message, duration: duration)
+          Config.server_middleware.invoke(instance, data, message) do
+            perform_job(instance, data: data, message: message)
+          end
 
           message.ack
           notify_batch(data, success: true)
           Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "done" }
-          true
+        rescue Requeue => e
+          message.nak(delay: Config.to_ns(e.delay))
+          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "requeue[#{e.delay}s]" }
         rescue Timeout::Error => e
           Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[timeout]" }
-          dropped = handle_failure(worker_class, message, data, e)
-          false if dropped
+          Cosmo.handle_error(e, error_context(message, data))
+          handle_failure(worker_class, message, data, e)
         rescue StandardError => e
           Logger.debug e
           Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[error]" }
-          dropped = handle_failure(worker_class, message, data, e)
-          false if dropped
+          Cosmo.handle_error(e, error_context(message, data))
+          handle_failure(worker_class, message, data, e)
         rescue Exception # rubocop:disable Lint/RescueException
           Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[exception]" }
           raise
         end
       ensure
-        Limit.instance.release(slot) if slot
         Logger.without(:jid)
         Logger.debug "processed message #{message.inspect}"
       end
@@ -157,27 +154,6 @@ module Cosmo
         end
       end
 
-      # Tries to acquire a concurrency slot for the job.
-      # Returns the slot key (String) on success, or false if all slots are
-      # taken (a message is NAK'd with a delay of +retry_in+ before returning
-      def acquire_concurrency_slot(worker_class, message, data)
-        options = worker_class.concurrency_options
-        key = worker_class.concurrency_key(data[:args])
-
-        slot = Limit.instance.acquire(key, jid: data[:jid], limit: options[:limit], duration: options[:duration])
-        return slot if slot
-
-        message.nak(delay: Config.to_ns(options[:retry_in]))
-        Logger.debug "concurrency limit reached for #{data[:class]}, re-queueing back #{data[:jid]}"
-        false
-      rescue NATS::Error => e
-        # Unexpected KV failure (e.g. transient NATS error). NAK immediately so
-        # the message is retried rather than stuck in-flight until ack_wait expires.
-        Logger.error e
-        message.nak
-        false
-      end
-
       def handle_failure(worker_class, message, data, exception) # rubocop:disable Naming/PredicateMethod
         current_attempt = message.metadata.num_delivered
         desired_retries = data[:retry].to_i + 1
@@ -231,6 +207,7 @@ module Cosmo
         delay.is_a?(Numeric) && delay.positive? ? delay : default_retry_delay(current_attempt)
       rescue StandardError => e
         Logger.error e
+        Cosmo.handle_error(e, { source: :retry_in, class: data[:class], jid: data[:jid] })
         default_retry_delay(current_attempt)
       end
 
@@ -249,16 +226,24 @@ module Cosmo
 
       def drop_message(message, data)
         message.term
-        Logger.debug "job dropped #{data[:jid]}"
+        Logger.debug "job dropped #{data&.dig(:jid)}"
       end
 
       # Logs why a message can't be processed at all and parks it in the DLQ.
       def reject_message(message, error, data = nil)
         Logger.error error
+        Cosmo.handle_error(error, error_context(message, data, source: :reject))
         move_message(message, data, error)
       end
 
+      def error_context(message, data, source: :job)
+        meta = message.metadata
+        { source:, stream: meta.stream, subject: message.subject, attempt: meta.num_delivered }.merge(Hash(data))
+      end
+
       def move_message(message, data = nil, exception = nil)
+        return drop_message(message, data) unless Config.dead.enabled
+
         klass = data ? Utils::String.underscore(data[:class]) : "default"
         headers = { "X-Stream" => message.metadata.stream, "X-Subject" => message.subject }
         headers.merge!(Failure.headers(exception)) if exception
@@ -276,7 +261,7 @@ module Cosmo
       def scheduled_by(message)
         header = message.header or return
 
-        header["X-Scheduled-By"] || header["Nats-Scheduler"]
+        header["X-Scheduled-By"] || header[NATS::JetStream::Header::SCHEDULER]
       end
 
       # A cron message is enqueued when NATS fires it, not when the scheduler dispatches it on, so
@@ -305,24 +290,13 @@ module Cosmo
         ENV.fetch("COSMO_JOBS_FETCH_TIMEOUT", 0.1).to_f
       end
 
-      def with_stats(message, &block)
-        API::Stats::Busy.instance.with(message) do
-          API::Stats::Totals.instance.with(&block)
-        end
-      end
-
       # @param job_instance [Cosmo::Job]
       # @param data [Hash]
       # @param message [NATS::Msg]
-      # @param duration [Float, nil]
       #
       # rubocop:disable-next Lint/UnusedMethodArgument
-      def perform_job(job_instance, data:, message:, duration: nil)
-        if duration
-          Timeout.timeout(duration, Timeout::Error, "execution expired after the #{duration}s duration limit") { job_instance.perform(*data[:args]) }
-        else
-          job_instance.perform(*data[:args])
-        end
+      def perform_job(job_instance, data:, message:)
+        job_instance.perform(*data[:args])
       end
     end
   end

data/lib/cosmo/job/requeue.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Job
+    # Raised by a middleware to put the job back on its stream for later: the processor naks it with +delay+ instead
+    # of failing it. The redelivery still counts as a delivery attempt toward the job's +retry+ and +max_deliver+.
+    # Raise it from a middleware that runs before the built-in stats middleware, or the attempt is counted as failed.
+    #
+    #   raise Cosmo::Job::Requeue.new(30) if maintenance?
+    class Requeue < StandardError
+      attr_reader :delay
+
+      # @param delay [Numeric] seconds before the job is delivered again
+      def initialize(delay = 0)
+        @delay = delay
+        super("requeued for #{delay}s")
+      end
+    end
+  end
+end

data/lib/cosmo/job.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "cosmo/job/data"
+require "cosmo/job/enqueuer"
+require "cosmo/job/requeue"
 require "cosmo/job/failure"
 require "cosmo/job/stream_filter"
 require "cosmo/job/limit"
@@ -96,30 +98,14 @@ module Cosmo
       end
 
       def perform(*args, async: true, **options)
-        batch = Batch.current if async
-        options[:batch_id] = batch.bid if batch
-        data = Data.new(name, args, default_options.merge(options))
-        unless async
-          payload = Utils::Json.parse(data.to_args[1])
-          raise ArgumentError, "Cannot parse payload" unless payload
-
-          new.perform(*payload[:args])
-          return
-        end
+        options = default_options.merge(options)
+        return Enqueuer.enqueue(name, args, options, batch: Batch.current) if async
 
-        publish(data, batch)
-      end
+        payload = Utils::Json.parse(Data.new(name, args, options).to_json)
+        raise ArgumentError, "Cannot parse payload" unless payload
 
-      # The batch is reserved a pending slot before we know the publish will
-      # succeed (must happen in that order -- see Batch#jobs). Roll it back
-      # if it never actually made it onto the stream, so the batch doesn't
-      # hang waiting for a completion that will never arrive.
-      def publish(data, batch)
-        batch&.register_job!
-        Publisher.publish_job(data)
-      rescue StandardError
-        batch&.rollback_job!
-        raise
+        new.perform(*payload[:args])
+        nil
       end
 
       def perform_async(*args)

data/lib/cosmo/logger.rb

@@ -61,7 +61,12 @@ module Cosmo
     class << self
       extend Forwardable
 
-      delegate %i[info error debug warn fatal trace] => :instance
+      delegate %i[info error debug warn fatal] => :instance
+    end
+
+    # Dropped when the logger has no TRACE level (a plain +::Logger+), rather than flooding its DEBUG.
+    def self.trace(...)
+      instance.trace(...) if instance.respond_to?(:trace)
     end
 
     def self.with(...)
@@ -82,10 +87,18 @@ module Cosmo
     def self.coerce_level(level)
       level.to_s.downcase == "trace" ? TRACE : ::Logger::Severity.coerce(level)
     end
-    private_class_method :coerce_level
 
+    # @param logger [::Logger] any stdlib-compatible logger. Its formatter is left alone, so context such as
+    #   +jid+ and +elapsed+ is only printed when it uses {SimpleFormatter}.
     def self.instance=(logger)
       @instance = logger
     end
+
+    # +COSMO_LOG_LEVEL+, when set, wins over +level+.
+    #
+    # @param level [Symbol, String, Integer] +:trace+, +:debug+, +:info+, +:warn+, +:error+, or +:fatal+
+    def self.level=(level)
+      instance.level = coerce_level(ENV.fetch("COSMO_LOG_LEVEL", level))
+    end
   end
 end

data/lib/cosmo/middleware/busy.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Middleware
+    # Lists the job on the Web UI's Busy page while it runs.
+    class Busy
+      # @param _job [Cosmo::Job]
+      # @param _data [Hash]
+      # @param message [NATS::Msg]
+      def call(_job, _data, message, &)
+        API::Stats::Busy.instance.with(message, &)
+      end
+    end
+  end
+end