cosmonats 0.7.0 → 0.9.0

data/lib/cosmo/middleware/chain.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Middleware
+    # An ordered list of middleware classes wrapped around job execution, outermost first.
+    #
+    # A middleware is any class whose instances respond to +call(job, data, message)+ and yield to run
+    # the rest of the chain. Every invocation builds fresh instances, so a middleware may keep per-job
+    # state in instance variables. Adding a class that is already in the chain moves it instead.
+    #
+    #   class Timing
+    #     def initialize(threshold:) = @threshold = threshold
+    #
+    #     def call(job, data, message)
+    #       yield
+    #     end
+    #   end
+    #
+    #   Cosmo.configure do |config|
+    #     config.server_middleware { |с| с.add Timing, threshold: 5 }
+    #   end
+    class Chain
+      include Enumerable
+
+      Entry = Struct.new(:klass, :args, :kwargs) do
+        def build
+          klass.new(*args, **kwargs)
+        end
+      end
+
+      # @yieldparam chain [Chain]
+      def initialize
+        @entries = []
+        yield self if block_given?
+      end
+
+      # @yieldparam entry [Entry]
+      def each(&)
+        @entries.each(&)
+      end
+
+      # Appends +klass+, so it runs innermost, closest to the job.
+      #
+      # @param klass [Class]
+      # @return [Chain]
+      def add(klass, *args, **kwargs)
+        remove(klass)
+        @entries << Entry.new(klass, args, kwargs)
+        self
+      end
+
+      # Inserts +klass+ first, so it runs outermost.
+      #
+      # @param klass [Class]
+      # @return [Chain]
+      def prepend(klass, *args, **kwargs)
+        remove(klass)
+        @entries.unshift(Entry.new(klass, args, kwargs))
+        self
+      end
+
+      # Inserts +klass+ right before +existing+, or first when +existing+ is not in the chain.
+      #
+      # @param existing [Class]
+      # @param klass [Class]
+      # @return [Chain]
+      def insert_before(existing, klass, *args, **kwargs)
+        entry = take(klass, args, kwargs)
+        @entries.insert(index(existing) || 0, entry)
+        self
+      end
+
+      # Inserts +klass+ right after +existing+, or last when +existing+ is not in the chain.
+      #
+      # @param existing [Class]
+      # @param klass [Class]
+      # @return [Chain]
+      def insert_after(existing, klass, *args, **kwargs)
+        entry = take(klass, args, kwargs)
+        position = index(existing)
+        @entries.insert(position ? position + 1 : @entries.size, entry)
+        self
+      end
+
+      # @param klass [Class]
+      # @return [Chain]
+      def remove(klass)
+        @entries.delete_if { _1.klass == klass }
+        self
+      end
+
+      # @param klass [Class]
+      # @return [Boolean]
+      def exists?(klass)
+        @entries.any? { _1.klass == klass }
+      end
+      alias include? exists?
+
+      # @return [Boolean]
+      def empty?
+        @entries.empty?
+      end
+
+      # @return [Chain]
+      def clear
+        @entries.clear
+        self
+      end
+
+      # Runs the block wrapped in every middleware, passing +args+ to each one's +call+.
+      #
+      # @return [Object] the outermost middleware's return value, normally the block's
+      def invoke(*args, &)
+        return yield if empty?
+
+        traverse(map(&:build), 0, args, &)
+      end
+
+      private
+
+      def traverse(chain, position, args, &block)
+        return yield if position >= chain.size
+
+        chain[position].call(*args) { traverse(chain, position + 1, args, &block) }
+      end
+
+      def index(klass)
+        @entries.index { _1.klass == klass }
+      end
+
+      def take(klass, args, kwargs)
+        position = index(klass)
+        position ? @entries.delete_at(position) : Entry.new(klass, args, kwargs)
+      end
+    end
+  end
+end

data/lib/cosmo/middleware/limit.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module Cosmo
+  module Middleware
+    # Enforces a job class's +limit:+ option: a concurrency slot per key, requeueing the job while every slot is
+    # taken, and the +duration+ timeout. Jobs without +limit:+ pass straight through. It runs first in the chain,
+    # so a requeued job never reaches the stats or user middleware.
+    class Limit
+      # @param job [Cosmo::Job]
+      # @param data [Hash]
+      # @param _message [NATS::Msg]
+      # @raise [Job::Requeue] while every concurrency slot is taken
+      def call(job, data, _message, &)
+        limit = job.class.default_options[:limit]
+        return yield unless limit
+
+        slot = acquire(job.class, data)
+        timeout(limit[:duration], &)
+      ensure
+        Job::Limit.instance.release(slot) if slot
+      end
+
+      private
+
+      def acquire(job_class, data)
+        options = job_class.concurrency_options
+        return unless options
+
+        key = job_class.concurrency_key(data[:args])
+        slot = Job::Limit.instance.acquire(key, jid: data[:jid], limit: options[:limit], duration: options[:duration])
+        raise Job::Requeue, options[:retry_in] unless slot
+
+        slot
+      rescue NATS::Error => e
+        Logger.error e
+        Cosmo.handle_error(e, { source: :limit, class: job_class.name, jid: data[:jid] })
+        raise Job::Requeue
+      end
+
+      def timeout(duration, &)
+        return yield unless duration
+
+        seconds = duration.to_i
+        Timeout.timeout(seconds, Timeout::Error, "execution expired after the #{seconds}s duration limit", &)
+      end
+    end
+  end
+end

data/lib/cosmo/middleware/metrics.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Middleware
+    # Records every execution for the web UI's Metrics tab while +config.metrics+ is enabled: its run time when it
+    # succeeds, a failure otherwise, and the wait from enqueue to the job's first delivery. ActiveJob jobs count under
+    # their own class.
+    class Metrics
+      # @param job [Cosmo::Job]
+      # @param data [Hash]
+      # @param _message [NATS::Msg]
+      def call(job, data, _message) # rubocop:disable Metrics/AbcSize
+        return yield unless Config.metrics.enabled
+
+        name = job_name(job, data)
+        wait_ms = (Time.now - job.enqueued_at) * 1000 if job.attempt == 1 && job.enqueued_at
+        started = clock
+        result = yield
+        API::Stats::Metrics.instance.record(name, exec_ms: clock - started, wait_ms:)
+        result
+      rescue Exception # rubocop:disable Lint/RescueException
+        API::Stats::Metrics.instance.record(name, wait_ms:, failed: true) if name
+        raise
+      end
+
+      private
+
+      def job_name(job, data)
+        return job.class.name unless defined?(ActiveJobAdapter::Executor) && job.is_a?(ActiveJobAdapter::Executor)
+
+        data.dig(:args, 0, :job_class).to_s
+      end
+
+      def clock
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
+      end
+    end
+  end
+end

data/lib/cosmo/{sentry/job_processor_middleware.rb → middleware/sentry.rb}

@@ -2,28 +2,39 @@
 
 # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize
 module Cosmo
-  module Sentry
-    module JobProcessorMiddleware
+  module Middleware
+    # Wraps every job execution in a Sentry transaction. Errors reach Sentry through {ERROR_HANDLER}, which also
+    # captures those outside jobs (stream batches, fetches, the scheduler). Needs +sentry-ruby+, so it is not loaded
+    # by default:
+    #
+    #   require "cosmo/middleware/sentry"
+    #
+    #   Cosmo.configure do |config|
+    #     config.server_middleware { |chain| chain.add Cosmo::Middleware::Sentry }
+    #     config.error_handlers << Cosmo::Middleware::Sentry::ERROR_HANDLER
+    #   end
+    class Sentry
       NAME_PREFIX = "Cosmonats"
       OP_NAME = "queue.cosmonats"
       SPAN_ORIGIN = "auto.queue.cosmonats"
       STATUS_OK = 200
       STATUS_FAIL = 500
 
-      # @param job_instance [Cosmo::Job]
+      ERROR_HANDLER = lambda do |error, context|
+        next unless ::Sentry.initialized?
+
+        ::Sentry.capture_exception(error, contexts: { cosmonats: context }, hint: { background: true, integration: "cosmonats" })
+      end
+
+      # @param job [Cosmo::Job]
       # @param data [Hash]
       # @param message [NATS::Msg]
-      # @param duration [Float, nil]
-      def perform_job(job_instance, data:, message:, duration: nil)
-        unless ::Sentry.initialized?
-          super
-          return
-        end
+      def call(job, data, message)
+        return yield unless ::Sentry.initialized?
 
         scope = ::Sentry.get_current_scope
-        transaction_name = "#{NAME_PREFIX}/#{job_instance.class.name}"
+        transaction_name = "#{NAME_PREFIX}/#{job.class.name}"
         scope.set_transaction_name(transaction_name, source: :task)
-
         transaction = ::Sentry.start_transaction(
           name: scope.transaction_name,
           source: scope.transaction_source,
@@ -35,29 +46,14 @@ module Cosmo
         transaction&.set_data("messaging.message.retry.count", data[:retry] || 0)
 
         begin
-          super
-
+          result = yield
           transaction&.set_http_status(STATUS_OK)
           transaction&.finish
-        rescue StandardError => e
-          ::Sentry.capture_exception(
-            e,
-            contexts: {
-              cosmonats: data.merge(
-                nats_stream: message.metadata.stream,
-                nats_subject: message.subject,
-                timeout_duration: duration
-              )
-            },
-            hint: {
-              background: true,
-              integration: "cosmonats"
-            }
-          )
+          result
+        rescue StandardError
           transaction&.set_http_status(STATUS_FAIL)
           transaction&.finish
-
-          raise e
+          raise
         end
       end
     end

data/lib/cosmo/middleware/totals.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Middleware
+    # Counts every execution as processed or failed, so a job retried twice and then succeeding adds two failures
+    # and one processed. A counter that cannot be written is logged rather than raised, so stats never fail a job.
+    class Totals
+      # @param _job [Cosmo::Job]
+      # @param _data [Hash]
+      # @param _message [NATS::Msg]
+      def call(_job, _data, _message)
+        result = yield
+        increment(:processed)
+        result
+      rescue Exception # rubocop:disable Lint/RescueException
+        increment(:failed)
+        raise
+      end
+
+      private
+
+      def increment(key)
+        API::Stats::Totals.instance.increment(key)
+      rescue StandardError => e
+        Logger.debug "Totals #{key} counter error: #{e.class} #{e.message}"
+      end
+    end
+  end
+end

data/lib/cosmo/middleware.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require "cosmo/middleware/chain"
+require "cosmo/middleware/limit"
+require "cosmo/middleware/busy"
+require "cosmo/middleware/totals"
+require "cosmo/middleware/metrics"

data/lib/cosmo/processor.rb

@@ -86,9 +86,10 @@ module Cosmo
           end
           all_paused = false
 
-          _, skip_t = consumer_state[stream_name]
+          key = backoff_key(config)
+          _, skip_t = consumer_state[key]
           if skip_t && Time.now < skip_t
-            Logger.trace "stream #{stream_name} is empty, backing off"
+            Logger.trace "#{key} is empty, backing off"
             next
           end
           all_empty = false
@@ -96,7 +97,7 @@ module Cosmo
           begin
             @pool.post do
               # Re-check, after possibly being blocked on post to thread pool
-              _, skip_t = consumer_state[stream_name]
+              _, skip_t = consumer_state[key]
               next if skip_t && Time.now < skip_t
 
               timeout = fetch_timeout(config)
@@ -105,11 +106,11 @@ module Cosmo
               messages = fetch(subscription, batch_size: config[:batch_size], timeout:)
               Logger.trace "fetched (#{messages&.size.to_i}) messages"
               if messages&.any?
-                consumer_state.delete(stream_name)
+                consumer_state.delete(key)
                 track_busy { process(messages, processor) }
               else
                 max_backoff = Utils::Duration.parse(ENV.fetch("COSMO_STREAM_EMPTY_BACKOFF_MAX", STREAM_EMPTY_BACKOFF_MAX))
-                consumer_state.compute(stream_name) do |(count, wake_at)|
+                consumer_state.compute(key) do |(count, wake_at)|
                   next [count, wake_at] if wake_at && wake_at > started_at
 
                   count = count.to_i + 1
@@ -179,6 +180,7 @@ module Cosmo
     rescue StandardError => e
       Logger.error "Snap! Error just happened"
       Logger.error "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
+      Cosmo.handle_error(e, { source: :fetch })
 
       backoff = ENV.fetch("COSMO_STREAMS_FETCH_BACKOFF", 5).to_f
       sleep([timeout, backoff].max) # backoff before retry
@@ -196,5 +198,9 @@ module Cosmo
     def consumer_state
       @consumer_state ||= Concurrent::Map.new
     end
+
+    def backoff_key(config)
+      config[:stream].to_s
+    end
   end
 end

data/lib/cosmo/publisher.rb

@@ -7,7 +7,7 @@ module Cosmo
     class << self
       extend Forwardable
 
-      delegate %i[publish publish_job publish_batch] => :instance
+      delegate %i[publish] => :instance
     end
 
     def self.instance
@@ -22,17 +22,5 @@ module Cosmo
       payload = (serializer || Stream::Serializer).serialize(data)
       @client.publish(subject, payload, **options)
     end
-
-    def publish_job(data)
-      subject, payload, params = data.to_args
-      @client.publish(subject, payload, **params)
-      data.jid
-    rescue NATS::JetStream::Error::NoStreamResponse
-      raise StreamNotFoundError, params[:stream].to_s
-    end
-
-    def publish_batch(subject, batch, **options)
-      batch.each { publish(subject, _1, **options) }
-    end
   end
 end

data/lib/cosmo/railtie.rb

@@ -12,10 +12,10 @@ module Cosmo
       require "cosmo/active_job"
     end
 
-    # Autoload config/cosmo.yml when it exists and no config has been loaded yet.
+    # Loads config/cosmo.yml, or the built-in defaults without one, unless a config has been loaded already.
     initializer "cosmo.load_config", after: "cosmo.active_job_adapter" do |app|
       config_path = app.root.join("config", "cosmo.yml")
-      Config.load(config_path.to_s) if config_path.exist? && Config.instance.none?
+      Config.load(config_path.exist? ? config_path.to_s : nil) if Config.instance.none?
     end
   end
 end

data/lib/cosmo/services.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Cosmo
+  # The streams Cosmo itself depends on. They are built from the settings in Cosmo.configure, never from cosmo.yml,
+  # and `cosmo --setup` creates them next to the user's streams.
+  module Services
+    SCHEDULED = Job::StreamFilter::SCHEDULED.to_s
+    DEAD = "dead"
+    METADATA = { "_cosmo.type" => "jobs" }.freeze
+    RETENTION_CHANGE = 10_052
+
+    module_function
+
+    # Holds delayed jobs and every cron template. It alone enables NATS message scheduling, since a schedule can only
+    # target a subject its own stream covers, and NATS refuses +discard: new+ wherever scheduling is enabled.
+    #
+    # @return [Hash, nil] nil when scheduling is turned off
+    def scheduled_stream
+      return unless Config.scheduled.enabled
+
+      { storage: "file", retention: "workqueue", discard: "old", allow_direct: true, allow_msg_schedules: true,
+        duplicate_window: Config.to_ns(120), subjects: ["jobs.#{SCHEDULED}.>", "#{API::Cron::Entry::SUBJECT_PREFIX}.>"],
+        num_replicas: Config.replicas, description: "Scheduled jobs and cron schedules", metadata: METADATA }
+    end
+
+    # +max_deliver+ stays above 1: the scheduler naks a message that is not yet due, and one whose dispatch failed,
+    # and both need another delivery.
+    #
+    # @return [Hash, nil] nil when scheduling is turned off
+    def scheduled_consumer
+      return unless Config.scheduled.enabled
+
+      { ack_policy: "explicit", max_deliver: 5, max_ack_pending: 100, ack_wait: 10, subject: "jobs.#{SCHEDULED}.>" }
+    end
+
+    # @return [Hash, nil] nil when dead-lettering is turned off
+    def dead_stream
+      dead = Config.dead
+      return unless dead.enabled
+
+      { storage: "file", retention: "workqueue", discard: "old", allow_direct: true, duplicate_window: Config.to_ns(120),
+        max_age: Config.to_ns(Utils::Duration.parse(dead.max_age)), max_msgs: dead.max_msgs, max_bytes: dead.max_bytes,
+        subjects: ["jobs.#{DEAD}.>"], num_replicas: Config.replicas, description: "Dead jobs", metadata: METADATA }
+    end
+
+    # Creates or updates every enabled service stream, plus the internal totals and batch counters and, while enabled,
+    # the job metrics.
+    #
+    # @return [Array<String>] the names of the service streams set up, without the internal ones
+    def setup!
+      streams = { SCHEDULED => scheduled_stream, DEAD => dead_stream }.compact
+      streams.each { |name, config| setup_stream(name, config) }
+      API::Stats::Totals.setup!
+      Batch::Counters.setup!
+      API::Stats::Metrics.setup! if Config.metrics.enabled
+      streams.keys
+    end
+
+    # @raise [Error] when an existing stream has another retention policy, which NATS can't change in place
+    def setup_stream(name, config)
+      Client.instance.setup_stream(name, config)
+    rescue NATS::JetStream::Error::APIError => e
+      raise unless e.err_code == RETENTION_CHANGE
+
+      raise Error, "The existing `#{name}` stream has another retention policy, which NATS can't change in place: " \
+                   "delete it (`nats stream rm #{name}`, dropping its messages) and rerun `cosmo --setup`"
+    end
+  end
+end

data/lib/cosmo/stream/processor.rb

@@ -38,8 +38,10 @@ module Cosmo
         processor.process(messages)
         Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "done" }
       rescue StandardError => e
-        Logger.debug e
+        Logger.error e
         Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
+        Cosmo.handle_error(e, { source: :stream, stream: metadata&.stream, processor: processor.class.name,
+                                size: messages&.size })
       rescue Exception # rubocop:disable Lint/RescueException
         Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
         raise
@@ -70,6 +72,10 @@ module Cosmo
         config.dig(:consumer, :subjects)
       end
 
+      def backoff_key(config)
+        config[:consumer_name] || config[:class].name
+      end
+
       def fetch_timeout(config)
         timeout = config[:fetch_timeout].to_f
         if timeout <= 0

data/lib/cosmo/utils/hash.rb

@@ -36,14 +36,13 @@ module Cosmo
         end
       end
 
-      # deep set
-      def set(hash, *keys, value)
-        last_key = keys.pop
-        target = keys.reduce(hash) do |base, key|
-          base[key] ||= {}
-          base[key]
-        end
-        target[last_key] = value
+      # Nested hashes merge recursively, +other+ winning; any other value in +other+ replaces the one in +base+.
+      #
+      # @param base [::Hash]
+      # @param other [::Hash]
+      # @return [::Hash] a new hash
+      def deep_merge(base, other)
+        base.merge(other) { |_, old, new| old.is_a?(::Hash) && new.is_a?(::Hash) ? deep_merge(old, new) : new }
       end
 
       # deep dup

data/lib/cosmo/utils.rb

@@ -4,7 +4,6 @@ require "cosmo/utils/hash"
 require "cosmo/utils/json"
 require "cosmo/utils/string"
 require "cosmo/utils/signal"
-require "cosmo/utils/warnings"
 require "cosmo/utils/duration"
 require "cosmo/utils/stopwatch"
 require "cosmo/utils/thread_pool"

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.7.0"
+  VERSION = "0.9.0"
 end