cosmonats 0.5.1 → 0.7.0

data/lib/cosmo/job/processor.rb

@@ -5,27 +5,62 @@ require "timeout"
 module Cosmo
   module Job
     class Processor < ::Cosmo::Processor
+      # Headers the scheduler consumes rather than forwards. The +X-+ ones address the dispatch
+      # itself, and NATS sets +Nats-Scheduler+/+Nats-Schedule-Next+ on a message it fires and
+      # rejects a publish that carries them back.
+      DISPATCH_HEADERS = %w[X-Stream X-Subject X-Execute-At Nats-Expected-Stream
+                            Nats-Scheduler Nats-Schedule-Next].freeze
+
+      # @raise [UnknownJobStreamError] when --streams/--stream or COSMO_JOBS_STREAMS names an unconfigured stream
+      def self.validate_options!(options)
+        StreamFilter.from(options[:streams]).validate!
+      end
+
+      # @return [Array<String>] job streams pulled from, plus +scheduled+ when dispatching scheduled jobs
+      def subscriptions
+        names = @consumers.map { |(_, config, _)| config[:stream].to_s }
+        names << StreamFilter::SCHEDULED.to_s if scheduler? && scheduled_config
+        names
+      end
+
       private
 
       def setup
+        filter = StreamFilter.from(@options[:streams]).validate!
+
         # Initialize singletons before starting to process messages
-        API::Busy.instance
-        API::Counter.instance
+        API::Stats::Busy.instance
+        API::Stats::Totals.instance
         Limit.instance
 
         jobs_config = Config.dig(:consumers, :jobs)
         jobs_config&.each do |stream_name, config|
-          next if stream_name == :scheduled # scheduled jobs are handled in schedule_loop
+          next if stream_name == StreamFilter::SCHEDULED # scheduled jobs are handled in schedule_loop
+          next unless filter.include?(stream_name)
 
           @consumers << subscribe(stream_name, config)
         end
+
+        log_subscriptions
+      end
+
+      def log_subscriptions
+        return if @consumers.empty?
+
+        names = @consumers.map { |(_, config, _)| config[:consumer] }
+        names << consumer_name(StreamFilter::SCHEDULED) if scheduler? && scheduled_config
+        Logger.info "subscribed: #{names.join(", ")}#{" (scheduler off)" unless scheduler?}"
+      end
+
+      def scheduled_config
+        Config.dig(:consumers, :jobs, StreamFilter::SCHEDULED)
       end
 
       def schedule_loop # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
-        config = Config.dig(:consumers, :jobs, :scheduled)
+        config = scheduled_config
         return unless config
 
-        subscription, = subscribe(:scheduled, config)
+        subscription, = subscribe(StreamFilter::SCHEDULED, config)
         while running?
           break unless running?
 
@@ -33,13 +68,18 @@ module Cosmo
           timeout = ENV.fetch("COSMO_JOBS_SCHEDULER_FETCH_TIMEOUT", 5).to_f
           messages = fetch(subscription, batch_size: 100, timeout:)
           messages&.each do |message|
-            headers = message.header.except("X-Stream", "X-Subject", "X-Execute-At", "Nats-Expected-Stream")
+            headers = message.header.except(*DISPATCH_HEADERS)
             stream, subject, execute_at = message.header.values_at("X-Stream", "X-Subject", "X-Execute-At")
             headers["Nats-Expected-Stream"] = stream
+            scheduler = message.header["Nats-Scheduler"]
+            if scheduler
+              headers["X-Scheduled-By"] = scheduler
+              headers["X-Enqueued-At"] = message.metadata.timestamp.to_f.to_s
+            end
             execute_at = execute_at.to_i
 
             if now >= execute_at
-              client.publish(subject, message.data, headers: headers)
+              client.publish(subject, message.data, header: headers)
               message.ack
             else
               message.nak(delay: Config.to_ns(execute_at - now))
@@ -60,16 +100,11 @@ module Cosmo
         message = messages.first
         Logger.debug "received messages #{messages.inspect}"
         data = Utils::Json.parse(message.data)
-        unless data
-          Logger.error ArgumentError.new("malformed payload")
-          move_message(message)
-          return
-        end
+        return reject_message(message, ArgumentError.new("malformed payload")) unless data
 
         worker_class = Utils::String.safe_constantize(data[:class])
         unless worker_class
-          Logger.error ArgumentError.new("#{data[:class]} class not found")
-          move_message(message, data)
+          reject_message(message, ArgumentError.new("#{data[:class]} class not found"), data)
           notify_batch(data, success: false)
           return
         end
@@ -115,9 +150,9 @@ module Cosmo
       def build_worker(worker_class, data, message)
         worker_class.new.tap do |worker|
           worker.jid = data[:jid]
-          worker.enqueued_at = message.metadata.timestamp
+          worker.enqueued_at = enqueued_at(message)
           worker.attempt = message.metadata.num_delivered
-          worker.scheduled_by = message.header&.dig("Nats-Scheduler")
+          worker.scheduled_by = scheduled_by(message)
           worker.batch_id = data[:batch_id]
         end
       end
@@ -154,7 +189,7 @@ module Cosmo
         end
 
         warn_capped(message, data, capped_at) if capped_at
-        data[:dead] ? move_message(message, data) : drop_message(message, data)
+        data[:dead] ? move_message(message, data, exception) : drop_message(message, data)
         notify_batch(data, success: false)
         true
       end
@@ -207,7 +242,7 @@ module Cosmo
         config = config.dup
         config[:batch_size] = 1
         config[:stream] = stream_name
-        config[:consumer] = "consumer-#{stream_name}"
+        config[:consumer] = consumer_name(stream_name)
         subscription = client.subscribe(config[:subject], config[:consumer], config.except(:subject, :priority, :stream, :batch_size, :consumer))
         [subscription, config, nil]
       end
@@ -217,16 +252,44 @@ module Cosmo
         Logger.debug "job dropped #{data[:jid]}"
       end
 
-      def move_message(message, data = nil)
+      # Logs why a message can't be processed at all and parks it in the DLQ.
+      def reject_message(message, error, data = nil)
+        Logger.error error
+        move_message(message, data, error)
+      end
+
+      def move_message(message, data = nil, exception = nil)
         klass = data ? Utils::String.underscore(data[:class]) : "default"
         headers = { "X-Stream" => message.metadata.stream, "X-Subject" => message.subject }
+        headers.merge!(Failure.headers(exception)) if exception
         Client.instance.publish("jobs.dead.#{klass}", message.data, header: headers)
         message.ack
         Logger.debug "job moved #{data&.dig(:jid)} to DLQ"
       end
 
       def scheduler?
-        true
+        @options.fetch(:scheduler, true)
+      end
+
+      # +Nats-Scheduler+ on a message NATS fired directly into a job stream, and +X-Scheduled-By+
+      # on one the scheduler dispatched onward, which may not carry the reserved header.
+      def scheduled_by(message)
+        header = message.header or return
+
+        header["X-Scheduled-By"] || header["Nats-Scheduler"]
+      end
+
+      # A cron message is enqueued when NATS fires it, not when the scheduler dispatches it on, so
+      # the firing forwards its own timestamp in +X-Enqueued-At+ to survive the re-publish.
+      def enqueued_at(message)
+        forwarded = message.header&.dig("X-Enqueued-At")
+
+        forwarded ? Time.at(forwarded.to_f) : message.metadata.timestamp
+      end
+
+      # Durable and per stream, so every process pulls from the same consumer and shares the work.
+      def consumer_name(stream_name)
+        "consumer-#{stream_name}"
       end
 
       def consumers
@@ -243,8 +306,8 @@ module Cosmo
       end
 
       def with_stats(message, &block)
-        API::Busy.instance.with(message) do
-          API::Counter.instance.with(&block)
+        API::Stats::Busy.instance.with(message) do
+          API::Stats::Totals.instance.with(&block)
         end
       end
 
@@ -253,15 +316,14 @@ module Cosmo
       # @param message [NATS::Msg]
       # @param duration [Float, nil]
       #
-      # rubocop:disable Lint/UnusedMethodArgument
+      # rubocop:disable-next Lint/UnusedMethodArgument
       def perform_job(job_instance, data:, message:, duration: nil)
         if duration
-          Timeout.timeout(duration) { job_instance.perform(*data[:args]) }
+          Timeout.timeout(duration, Timeout::Error, "execution expired after the #{duration}s duration limit") { job_instance.perform(*data[:args]) }
         else
           job_instance.perform(*data[:args])
         end
       end
-      # rubocop:enable Lint/UnusedMethodArgument
     end
   end
 end

data/lib/cosmo/job/stream_filter.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Job
+    # Which job streams a worker subscribes to: every configured stream, or the subset named by
+    # +--streams+/+--stream+, falling back to the +COSMO_JOBS_STREAMS+ environment variable.
+    #
+    # The scheduled stream is a service stream. Naming it selects nothing, it runs unless the worker
+    # was started with +--no-scheduler+.
+    class StreamFilter
+      SCHEDULED = :scheduled
+      ENV_NAME = "COSMO_JOBS_STREAMS"
+
+      # @param names [Array<String>, nil] stream names from the command line
+      # @return [StreamFilter]
+      def self.from(names)
+        new(names || ENV[ENV_NAME]&.split(","))
+      end
+
+      # @param names [Array<String>, nil]
+      def initialize(names)
+        @names = Array(names).map(&:strip).reject(&:empty?)
+      end
+
+      # @return [Boolean] whether every configured stream is selected
+      def all?
+        @names.empty?
+      end
+
+      # @param stream_name [String, Symbol]
+      # @return [Boolean] whether the stream is selected
+      def include?(stream_name)
+        all? || @names.include?(stream_name.to_s)
+      end
+
+      # @return [StreamFilter] self, so it can be chained onto {.from}
+      # @raise [UnknownJobStreamError] when a name is not configured
+      def validate!
+        return self if all?
+
+        unknown = @names - configured - [SCHEDULED.to_s]
+        raise UnknownJobStreamError.new(unknown, configured) if unknown.any?
+
+        self
+      end
+
+      private
+
+      def configured
+        @configured ||= Config.dig(:consumers, :jobs)&.keys.to_a.map(&:to_s) - [SCHEDULED.to_s]
+      end
+    end
+  end
+end

data/lib/cosmo/job.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "cosmo/job/data"
+require "cosmo/job/failure"
+require "cosmo/job/stream_filter"
 require "cosmo/job/limit"
 require "cosmo/job/processor"
 

data/lib/cosmo/logger.rb

@@ -5,6 +5,8 @@ require "forwardable"
 
 module Cosmo
   module Logger
+    TRACE = -1 # Below DEBUG; opt in with COSMO_LOG_LEVEL=trace for high-frequency polling/loop noise
+
     module Context
       KEY = :cosmo_context
 
@@ -45,10 +47,21 @@ module Cosmo
       end
     end
 
+    # Adds TRACE (below DEBUG) on top of stdlib's fixed DEBUG..UNKNOWN severities.
+    class Instance < ::Logger
+      def trace(progname = nil, &)
+        add(TRACE, nil, progname, &)
+      end
+
+      def format_severity(severity)
+        severity == TRACE ? "TRACE" : super
+      end
+    end
+
     class << self
       extend Forwardable
 
-      delegate %i[info error debug warn fatal] => :instance
+      delegate %i[info error debug warn fatal trace] => :instance
     end
 
     def self.with(...)
@@ -60,12 +73,17 @@ module Cosmo
     end
 
     def self.instance
-      @instance ||= ::Logger.new($stdout).tap do |logger|
+      @instance ||= Instance.new($stdout).tap do |logger|
         logger.formatter = SimpleFormatter.new
-        logger.level = ::Logger::Severity.coerce(ENV.fetch("COSMO_LOG_LEVEL", "info"))
+        logger.level = coerce_level(ENV.fetch("COSMO_LOG_LEVEL", "info"))
       end
     end
 
+    def self.coerce_level(level)
+      level.to_s.downcase == "trace" ? TRACE : ::Logger::Severity.coerce(level)
+    end
+    private_class_method :coerce_level
+
     def self.instance=(logger)
       @instance = logger
     end

data/lib/cosmo/processor.rb

@@ -5,20 +5,29 @@ module Cosmo
     STREAM_PAUSED_RECHECK_TTL = "5s" # How long a stream's paused state is cached before re-checking (override via COSMO_STREAM_PAUSED_RECHECK_TTL)
     STREAMS_PAUSED_IDLE_SLEEP = "1s" # How long to sleep when every stream is paused, preventing a tight CPU spin (override via COSMO_STREAMS_PAUSED_IDLE_SLEEP)
     STREAM_EMPTY_BACKOFF_MAX = "5s"  # Max sleep between empty fetches (override via COSMO_STREAM_EMPTY_BACKOFF_MAX)
+    QUIET_SLEEP = "5s"               # How long to sleep between checks while in quiet mode (override via COSMO_QUIET_IDLE_SLEEP)
 
     def self.run(...)
       new(...).tap(&:run)
     end
 
+    # Checks the command-line options this processor understands, before anything is booted or connected.
+    #
+    # @param options [Hash]
+    # @return [void]
+    def self.validate_options!(_options); end
+
     attr_reader :consumers
 
-    def initialize(pool, running, options)
+    def initialize(pool, running, options, quiet:)
       @pool = pool
       @running = running
+      @quiet = quiet
       @options = options
       @threads = []
       @consumers = []
       @cache = Utils::TTLCache.new
+      @busy = Concurrent::AtomicFixnum.new
     end
 
     def run
@@ -29,6 +38,13 @@ module Cosmo
       run_loop
     end
 
+    # Threads processing fetched messages right now; threads waiting on a fetch don't count.
+    #
+    # @return [Integer]
+    def busy
+      @busy.value
+    end
+
     def stop(timeout = Config[:timeout])
       @running.make_false
       @pool.shutdown
@@ -52,6 +68,11 @@ module Cosmo
       while running?
         break if shutdown
 
+        if quiet?
+          sleep(Utils::Duration.parse(ENV.fetch("COSMO_QUIET_IDLE_SLEEP", QUIET_SLEEP)))
+          next
+        end
+
         all_empty = true  # every stream is empty
         all_paused = true # every stream is paused
         consumers.each do |(subscription, config, processor)| # rubocop:disable Metrics/BlockLength
@@ -60,14 +81,14 @@ module Cosmo
           stream_name = config[:stream].to_s
           ttl = Utils::Duration.parse(ENV.fetch("COSMO_STREAM_PAUSED_RECHECK_TTL", STREAM_PAUSED_RECHECK_TTL))
           if @cache.fetch("#{stream_name}:paused", ttl:) { API::Stream.new(stream_name).paused? }
-            Logger.debug "stream #{stream_name} is paused, skipping fetch"
+            Logger.trace "stream #{stream_name} is paused, skipping fetch"
             next
           end
           all_paused = false
 
           _, skip_t = consumer_state[stream_name]
           if skip_t && Time.now < skip_t
-            Logger.debug "stream #{stream_name} is empty, backing off"
+            Logger.trace "stream #{stream_name} is empty, backing off"
             next
           end
           all_empty = false
@@ -79,16 +100,19 @@ module Cosmo
               next if skip_t && Time.now < skip_t
 
               timeout = fetch_timeout(config)
-              Logger.debug "fetching #{fetch_subjects(config).inspect}, timeout=#{timeout}"
-              messages = lock(stream_name) { fetch(subscription, batch_size: config[:batch_size], timeout:) }
-              Logger.debug "fetched (#{messages&.size.to_i}) messages"
+              Logger.trace "fetching #{fetch_subjects(config).inspect}, timeout=#{timeout}"
+              started_at = Time.now
+              messages = fetch(subscription, batch_size: config[:batch_size], timeout:)
+              Logger.trace "fetched (#{messages&.size.to_i}) messages"
               if messages&.any?
                 consumer_state.delete(stream_name)
-                process(messages, processor)
+                track_busy { process(messages, processor) }
               else
                 max_backoff = Utils::Duration.parse(ENV.fetch("COSMO_STREAM_EMPTY_BACKOFF_MAX", STREAM_EMPTY_BACKOFF_MAX))
-                consumer_state.compute(stream_name) do |current|
-                  count = (current&.first || 0) + 1
+                consumer_state.compute(stream_name) do |(count, wake_at)|
+                  next [count, wake_at] if wake_at && wake_at > started_at
+
+                  count = count.to_i + 1
                   backoff = [timeout * (2**(count - 1)), max_backoff].min
                   [count, Time.now + backoff]
                 end
@@ -104,14 +128,14 @@ module Cosmo
 
         if all_paused
           period = Utils::Duration.parse(ENV.fetch("COSMO_STREAMS_PAUSED_IDLE_SLEEP", STREAMS_PAUSED_IDLE_SLEEP))
-          Logger.debug "all streams paused, sleep=#{period}"
+          Logger.trace "all streams paused, sleep=#{period}"
           sleep(period)
         elsif all_empty
           next_wake = consumer_state.values.filter_map { |_, t| t }.min
           next unless next_wake # entry was deleted concurrently (messages arrived), re-loop immediately
 
           remaining = [next_wake - Time.now, 0.01].max
-          Logger.debug "all streams empty, sleep=#{remaining}"
+          Logger.trace "all streams empty, sleep=#{remaining}"
           sleep(remaining)
         end
       end
@@ -129,10 +153,21 @@ module Cosmo
       raise NotImplementedError
     end
 
+    def track_busy
+      @busy.increment
+      yield
+    ensure
+      @busy.decrement
+    end
+
     def running?
       @running.true?
     end
 
+    def quiet?
+      @quiet.true?
+    end
+
     def scheduler?
       false
     end
@@ -158,11 +193,6 @@ module Cosmo
       Utils::Stopwatch.new
     end
 
-    def lock(stream_name, &)
-      @locks ||= Hash.new { |h, k| h[k] = Mutex.new }
-      @locks[stream_name].synchronize(&)
-    end
-
     def consumer_state
       @consumer_state ||= Concurrent::Map.new
     end

data/lib/cosmo/sentry/job_processor_middleware.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+# rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize
 module Cosmo
   module Sentry
     module JobProcessorMiddleware
@@ -63,4 +63,3 @@ module Cosmo
     end
   end
 end
-# rubocop:enable Metrics/MethodLength, Metrics/AbcSize

data/lib/cosmo/stream/message.rb

@@ -7,8 +7,9 @@ module Cosmo
     class Message
       extend Forwardable
 
-      delegate %i[subject reply header metadata ack nack term in_progress] => :@msg
+      delegate %i[subject reply header metadata ack nak term in_progress] => :@msg
       delegate %i[timestamp num_delivered num_pending] => :metadata
+      alias nack nak
 
       def initialize(msg, serializer: nil)
         @msg = msg

data/lib/cosmo/stream/processor.rb

@@ -3,6 +3,11 @@
 module Cosmo
   module Stream
     class Processor < ::Cosmo::Processor
+      # @return [Array<String>] names of the stream classes being processed
+      def subscriptions
+        @consumers.map { |(_, config, _)| config[:class].name }
+      end
+
       private
 
       def setup

data/lib/cosmo/utils/json.rb

@@ -8,7 +8,7 @@ module Cosmo
       module_function
 
       def parse(value, default: nil, symbolize_names: true, **options)
-        JSON.parse(value, options.merge(symbolize_names:))
+        JSON.parse(value, **options, symbolize_names:)
       rescue TypeError, JSON::ParserError
         default
       end