cosmonats 0.3.0 → 0.4.1

data/lib/cosmo/config.rb

@@ -31,11 +31,20 @@ module Cosmo
       end
 
       config[:setup]&.each_key do |type|
+        next if type == :cron
+
         config[:setup][type]&.each_key do |name|
           c = config[:setup][type][name]
           c[:max_age] = c[:max_age].to_i * NANO if c[:max_age]
           c[:duplicate_window] = c[:duplicate_window].to_i * NANO if c[:duplicate_window]
           c[:subjects] = c[:subjects].map { |s| format(s, name: name) } if c[:subjects]
+
+          next unless type == :jobs # Every jobs stream supports NATS 2.14 message scheduling.
+
+          c[:allow_msg_schedules] = true
+          cron_subject = "#{API::Cron::Entry::SUBJECT_PREFIX}.#{name}.>"
+          c[:subjects] = Array(c[:subjects])
+          c[:subjects] << cron_subject unless c[:subjects].include?(cron_subject)
         end
       end
     end

data/lib/cosmo/job/data.rb

@@ -5,7 +5,7 @@ require "json"
 module Cosmo
   module Job
     class Data
-      DEFAULTS = { stream: :default, retry: 3, dead: true }.freeze
+      DEFAULTS = { stream: :default, retry: 3, dead: true, limit: nil }.freeze
 
       attr_reader :jid
 

data/lib/cosmo/job/limit.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Job
+    # Distributed concurrency limiter backed by NATS Key-Value with per-message TTL.
+    #
+    # Each unit of concurrency is a numbered KV slot:
+    #   "{concurrency_key}/0", "{concurrency_key}/1", ..., "{concurrency_key}/{limit-1}"
+    #
+    # Acquiring a slot is a single atomic `set` (CAS with last-revision=0).
+    # Only one worker can win a given slot; losers try the next number.
+    # When a job finishes the slot is deleted; if the worker crashes NATS
+    # expires it automatically via the per-message Nats-TTL header.
+    class Limit
+      BUCKET = "cosmo_jobs_limits"
+
+      def self.instance
+        @instance ||= new
+      end
+
+      def initialize
+        @kv = API::KV.new(BUCKET, allow_msg_ttl: true)
+      end
+
+      # Try to acquire one of the numbered slots for +key+.
+      #
+      # @param key      [String]  concurrency key
+      # @param jid      [String]  stored as the slot value for observability
+      # @param limit    [Integer] number of slots (0 … limit-1)
+      # @param duration [Integer] seconds before the slot is auto-expired by NATS
+      # @return [String, nil] the acquired slot key, or nil when all slots are taken
+      def acquire(key, jid:, limit:, duration:)
+        0.upto(limit - 1) do |i|
+          slot = "#{key}/#{i}"
+          @kv.set(slot, jid, ttl: duration)
+          return slot
+        rescue NATS::KeyValue::KeyWrongLastSequenceError
+          next # slot is live, try the next one
+        end
+        nil # all slots occupied
+      end
+
+      # Release a previously acquired slot.
+      def release(slot)
+        @kv.delete(slot)
+      rescue NATS::Error
+        # best effort — slot TTL will reclaim it if delete fails
+      end
+    end
+  end
+end

data/lib/cosmo/job/processor.rb

@@ -1,11 +1,18 @@
 # frozen_string_literal: true
 
+require "timeout"
+
 module Cosmo
   module Job
     class Processor < ::Cosmo::Processor
       private
 
       def setup
+        # Initialize singletons before starting to process messages
+        API::Busy.instance
+        API::Counter.instance
+        Limit.instance
+
         jobs_config = Config.dig(:consumers, :jobs)
         jobs_config&.each do |stream_name, config|
           next if stream_name == :scheduled # scheduled jobs are handled in schedule_loop
@@ -44,7 +51,7 @@ module Cosmo
         end
       end
 
-      def process(messages, _) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def process(messages, _) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         message = messages.first
         Logger.debug "received messages #{messages.inspect}"
         data = Utils::Json.parse(message.data)
@@ -61,30 +68,64 @@ module Cosmo
           return
         end
 
+        if worker_class.limits_concurrency?
+          slot = acquire_concurrency_slot(worker_class, message, data)
+          return if slot == false
+        end
+
+        duration = worker_class.default_options[:limit]&.dig(:duration)&.to_i
+
         with_stats(message) do
           sw = stopwatch
           Logger.with(jid: data[:jid])
           Logger.info "start"
-          instance = worker_class.new
-          instance.jid = data[:jid]
-          instance.perform(*data[:args])
+
+          instance = worker_class.new.tap { |w| w.jid = data[:jid] }
+          perform_job(instance, data: data, message: message, duration: duration)
+
           message.ack
           Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "done" }
           true
+        rescue Timeout::Error
+          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[timeout]" }
+          dropped = handle_failure(message, data)
+          false if dropped
         rescue StandardError => e
           Logger.debug e
-          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
+          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[error]" }
           dropped = handle_failure(message, data)
           false if dropped
         rescue Exception # rubocop:disable Lint/RescueException
-          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
+          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[exception]" }
           raise
         end
       ensure
+        Limit.instance.release(slot) if slot
         Logger.without(:jid)
         Logger.debug "processed message #{message.inspect}"
       end
 
+      # Tries to acquire a concurrency slot for the job.
+      # Returns the slot key (String) on success, or false if all slots are
+      # taken (message is NAK'd with a delay equal to +duration+ before returning).
+      def acquire_concurrency_slot(worker_class, message, data)
+        options = worker_class.concurrency_options
+        key = worker_class.concurrency_key(data[:args])
+
+        slot = Limit.instance.acquire(key, jid: data[:jid], limit: options[:limit], duration: options[:duration])
+        return slot if slot
+
+        message.nak(delay: options[:duration] * Config::NANO)
+        Logger.debug "concurrency limit reached for #{data[:class]}, re-queueing back #{data[:jid]}"
+        false
+      rescue NATS::Error => e
+        # Unexpected KV failure (e.g. transient NATS error). NAK immediately so
+        # the message is retried rather than stuck in-flight until ack_wait expires.
+        Logger.error e
+        message.nak
+        false
+      end
+
       def handle_failure(message, data) # rubocop:disable Naming/PredicateMethod
         current_attempt = message.metadata.num_delivered
         max_retries = data[:retry].to_i + 1
@@ -93,7 +134,7 @@ module Cosmo
           # NATS will auto-retry with delay (exponential backoff based on current attempt).
           # When max_deliver is reached, NATS stops redelivering the message and marks it as "max deliveries exceeded".
           # The message is effectively abandoned by NATS — it stays in the stream (consuming a slot) but will never be delivered again to that consumer.
-          delay_ns = ((current_attempt**4) + 15) * 1_000_000_000
+          delay_ns = ((current_attempt**4) + 15) * Config::NANO
           message.nak(delay: delay_ns)
           return false
         end
@@ -146,6 +187,21 @@ module Cosmo
           API::Counter.instance.with(&block)
         end
       end
+
+      # @param job_instance [Cosmo::Job]
+      # @param data [Hash]
+      # @param message [NATS::Msg]
+      # @param duration [Float, nil]
+      #
+      # rubocop:disable Lint/UnusedMethodArgument
+      def perform_job(job_instance, data:, message:, duration: nil)
+        if duration
+          Timeout.timeout(duration) { job_instance.perform(*data[:args]) }
+        else
+          job_instance.perform(*data[:args])
+        end
+      end
+      # rubocop:enable Lint/UnusedMethodArgument
     end
   end
 end

data/lib/cosmo/job.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "cosmo/job/data"
+require "cosmo/job/limit"
 require "cosmo/job/processor"
 
 module Cosmo
@@ -10,8 +11,56 @@ module Cosmo
     end
 
     module ClassMethods
-      def options(stream: nil, retry: nil, dead: nil)
-        default_options.merge!({ stream:, retry:, dead: }.compact)
+      # @option config [Symbol]  :stream  NATS stream to publish to (default: :default)
+      # @option config [Integer] :retry   max delivery attempts before giving up (default: 3)
+      # @option config [Boolean] :dead    move to dead-letter stream after retries exhausted (default: true)
+      # @option config [Hash]    :limit   execution limits:
+      #
+      #   limit: { duration: 30 }
+      #   limit: { duration: 30, concurrency: 3 }
+      #   limit: { duration: 30, concurrency: { to: 3, key: ->(id) { id } } }
+      #
+      # @option config [Integer] :"limit[:duration]"    hard execution timeout in seconds. The job thread is
+      #   killed after this many seconds and counts as a failed attempt (retried with exponential backoff,
+      #   moved to DLQ after retries exhausted).
+      # @option config [Integer, Hash] :"limit[:concurrency]"  caps how many instances run at once across all
+      #   workers. Jobs that cannot acquire a slot are NAK'd with a delay equal to +duration+ so they are not
+      #   re-delivered until the slot is guaranteed free. Requires +duration+.
+      #   Pass an Integer for a class-wide cap, or <tt>{ to: N, key: ->(args) {} }</tt> to scope per key.
+      def options(**config)
+        if config[:limit] && config.dig(:limit, :concurrency) && !config.dig(:limit, :duration).to_i.positive?
+          raise ArgumentError, "limit: duration is required when concurrency is set"
+        end
+
+        default_options.merge!(config)
+      end
+      alias cosmo_options options
+
+      def limits_concurrency?
+        !!concurrency_options
+      end
+
+      # Returns a normalized concurrency config hash, or +nil+ when not configured.
+      # Always contains +:limit+, +:key+, and +:duration+.
+      def concurrency_options
+        value = default_options.dig(:limit, :concurrency)
+        duration = default_options.dig(:limit, :duration).to_i
+        return unless value
+
+        case value
+        when Integer then { limit: value, key: nil, duration: duration }
+        when Hash    then { limit: value.fetch(:to), key: value[:key], duration: duration }
+        end
+      end
+
+      # Derive the fully-scoped concurrency key for a given args array.
+      def concurrency_key(args)
+        config = concurrency_options
+        return unless config
+
+        base = Utils::String.underscore(name)
+        suffix = config[:key]&.call(*args)
+        suffix ? "#{base}/#{suffix}" : base
       end
 
       def perform(*args, async: true, **options)

data/lib/cosmo/processor.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Cosmo
-  class Processor # rubocop:disable Metrics/ClassLength
+  class Processor
     STREAM_PAUSED_RECHECK_TTL = 5.0 # Seconds a stream's paused state is cached before re-checking (override via COSMO_STREAM_PAUSED_RECHECK_TTL)
     STREAMS_PAUSED_IDLE_SLEEP = 1.0 # Seconds to sleep when every stream is paused, preventing a tight CPU spin (override via COSMO_STREAMS_PAUSED_IDLE_SLEEP)
     STREAM_EMPTY_BACKOFF_MAX = 5.0  # Max seconds to sleep between empty fetches (override via COSMO_STREAM_EMPTY_BACKOFF_MAX)

data/lib/cosmo/railtie.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Cosmo
+  # Rails Railtie — loaded automatically when cosmonats is required inside a
+  # Rails application. Ensures the ActiveJob adapter constant is registered
+  # before Rails tries to resolve it and autoloads +config/cosmo.yml+ when
+  # the file is present.
+  class Railtie < ::Rails::Railtie
+    # Make Cosmo::ActiveJobAdapter::Adapter available under the conventional
+    # ActiveJob namespace so :cosmonats resolves without any extra requires.
+    initializer "cosmo.active_job_adapter", before: :run_prepare_callbacks do
+      require "cosmo/active_job"
+    end
+
+    # Autoload config/cosmo.yml when it exists and no config has been loaded yet.
+    initializer "cosmo.load_config", after: "cosmo.active_job_adapter" do |app|
+      config_path = app.root.join("config", "cosmo.yml")
+      Config.load(config_path.to_s) if config_path.exist? && Config.instance.none?
+    end
+  end
+end

data/lib/cosmo/sentry/auto.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "cosmo/job/processor"
+require "cosmo/sentry/job_processor_middleware"
+
+Cosmo::Job::Processor.prepend Cosmo::Sentry::JobProcessorMiddleware

data/lib/cosmo/sentry/job_processor_middleware.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+module Cosmo
+  module Sentry
+    module JobProcessorMiddleware
+      NAME_PREFIX = "Cosmonats"
+      OP_NAME = "queue.cosmonats"
+      SPAN_ORIGIN = "auto.queue.cosmonats"
+      STATUS_OK = 200
+      STATUS_FAIL = 500
+
+      # @param job_instance [Cosmo::Job]
+      # @param data [Hash]
+      # @param message [NATS::Msg]
+      # @param duration [Float, nil]
+      def perform_job(job_instance, data:, message:, duration: nil)
+        unless ::Sentry.initialized?
+          super
+          return
+        end
+
+        scope = ::Sentry.get_current_scope
+        transaction_name = "#{NAME_PREFIX}/#{job_instance.class.name}"
+        scope.set_transaction_name(transaction_name, source: :task)
+
+        transaction = ::Sentry.start_transaction(
+          name: scope.transaction_name,
+          source: scope.transaction_source,
+          op: OP_NAME,
+          origin: SPAN_ORIGIN
+        )
+        transaction&.set_data("messaging.message.id", data[:jid])
+        transaction&.set_data("messaging.destination.name", "#{message.metadata.stream}:#{message.subject}")
+        transaction&.set_data("messaging.message.retry.count", data[:retry] || 0)
+
+        begin
+          super
+
+          transaction&.set_http_status(STATUS_OK)
+          transaction&.finish
+        rescue StandardError => e
+          ::Sentry.capture_exception(
+            e,
+            contexts: {
+              cosmonats: data.merge(
+                nats_stream: message.metadata.stream,
+                nats_subject: message.subject,
+                timeout_duration: duration
+              )
+            },
+            hint: {
+              background: true,
+              integration: "cosmonats"
+            }
+          )
+          transaction&.set_http_status(STATUS_FAIL)
+          transaction&.finish
+
+          raise e
+        end
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/MethodLength, Metrics/AbcSize

data/lib/cosmo/stream/processor.rb

@@ -17,7 +17,7 @@ module Cosmo
         @configs.each { @consumers << subscribe(nil, _1) }
       end
 
-      def process(messages, processor) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def process(messages, processor) # rubocop:disable Metrics/AbcSize
         metadata = messages.last.metadata
         serializer = processor.class.default_options.dig(:publisher, :serializer)
         messages = messages.map { Message.new(_1, serializer:) }
@@ -49,7 +49,7 @@ module Cosmo
       end
 
       def dynamic_config
-        Config.internal[:streams].map { _1.default_options.merge(class: _1) }
+        Config.internal[:streams]&.map { _1.default_options.merge(class: _1) }.to_a
       end
 
       def subscribe(_stream_name, config)

data/lib/cosmo/stream.rb

@@ -12,10 +12,11 @@ module Cosmo
     end
 
     module ClassMethods
-      def options(stream: nil, consumer_name: nil, batch_size: nil, fetch_timeout: nil, start_position: nil, consumer: nil, publisher: nil) # rubocop:disable Metrics/ParameterLists
+      def options(stream: nil, consumer_name: nil, batch_size: nil, fetch_timeout: nil, start_position: nil, consumer: nil, publisher: nil)
         register
         default_options.merge!({ stream:, consumer_name:, batch_size:, fetch_timeout:, start_position:, consumer:, publisher: }.compact)
       end
+      alias cosmo_options options
 
       def publish(data, subject: nil, **options)
         stream = default_options[:stream]

data/lib/cosmo/utils/hash.rb

@@ -23,6 +23,19 @@ module Cosmo
         end
       end
 
+      def stringify_keys(obj)
+        case obj
+        when ::Hash
+          obj.each_with_object({}) do |(key, value), result|
+            result[key.to_s] = stringify_keys(value)
+          end
+        when ::Array
+          obj.map { |v| stringify_keys(v) }
+        else
+          obj
+        end
+      end
+
       # deep set
       def set(hash, *keys, value)
         last_key = keys.pop

data/lib/cosmo/utils/overrides.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 Cosmo::Utils::Warnings.silence do
-  members = NATS::JetStream::API::StreamConfig.members + [:allow_msg_counter]
+  members = NATS::JetStream::API::StreamConfig.members + %i[allow_msg_counter allow_msg_schedules]
   NATS::JetStream::API::StreamConfig = Struct.new(*members, keyword_init: true) do
     def initialize(opts = {})
       rem = opts.keys - members

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.3.0"
+  VERSION = "0.4.1"
 end

data/lib/cosmo/web/assets/app.css

@@ -292,6 +292,34 @@ section > header {
   z-index: auto;
 }
 
+/* ── Section tabs ───────────────────────────────────────────────────────── */
+.tabs {
+  display: flex;
+  gap: 0;
+  border-bottom: 2px solid var(--color-border);
+  margin-bottom: var(--space-3x);
+}
+
+.tabs a {
+  padding: var(--space) var(--space-3x);
+  text-decoration: none;
+  color: var(--color-text-light);
+  border-bottom: 2px solid transparent;
+  margin-bottom: -2px;
+  font-weight: 500;
+  transition: color 0.15s, border-color 0.15s;
+}
+
+.tabs a:hover {
+  color: var(--color-text);
+  border-bottom-color: var(--color-border);
+}
+
+.tabs a.active {
+  color: var(--color-primary);
+  border-bottom-color: var(--color-primary);
+}
+
 section .nav {
   display: flex;
   gap: var(--space);
@@ -448,6 +476,20 @@ time {
 .stream-link { color: var(--color-primary); font-weight: 600; }
 .stream-link:hover { text-decoration: underline; }
 
+/* ── Pagination ─────────────────────────────────────────────────────────── */
+.pagination {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: var(--space);
+  padding: var(--space-2x) 0 var(--space);
+}
+.btn-disabled {
+  opacity: 0.4;
+  cursor: default;
+  pointer-events: none;
+}
+
 /* ── Actions ───────────────────────────────────────────────────────────── */
 .actions-form { display: flex; flex-direction: column; gap: var(--space-1-2); }
 
@@ -467,6 +509,23 @@ details code { display: block; max-width: 400px; white-space: pre-wrap; }
 .htmx-request .htmx-indicator,
 .htmx-request.htmx-indicator   { opacity: 1; }
 
+#global-spinner {
+  position: fixed;
+  top: var(--space-2x);
+  right: var(--space-2x);
+  width: 24px;
+  height: 24px;
+  border: 3px solid var(--color-border);
+  border-top-color: var(--color-primary);
+  border-radius: 50%;
+  animation: spin 0.6s linear infinite;
+  z-index: 100;
+}
+
+@keyframes spin {
+  to { transform: rotate(360deg); }
+}
+
 /* ── Responsive ────────────────────────────────────────────────────────── */
 @media (max-width: 768px) {
   .nav            { flex-wrap: wrap; }

data/lib/cosmo/web/controllers/crons.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "cosmo/web/controllers/application"
+
+module Cosmo
+  class Web
+    module Controllers
+      class Crons < Application
+        def index
+          content_for :title, "Crons"
+          ok render("crons/index", layout: true)
+        end
+
+        def _table
+          ok render("crons/_table", { schedules: cron.all })
+        end
+
+        # Dispatch the job immediately, bypassing the schedule timer.
+        # Expects params["subject"] = the schedule subject stored in NATS.
+        def run_now
+          subject = Rack::Utils.unescape(params["subject"].to_s)
+          cron.run_now!(subject)
+          ok
+        end
+
+        # Purge the schedule from NATS so it stops firing.
+        def delete
+          subject = Rack::Utils.unescape(params["subject"].to_s)
+          cron.delete!(subject)
+          _table
+        end
+
+        private
+
+        def cron
+          @cron ||= API::Cron.instance
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/web/controllers/jobs.rb

@@ -72,12 +72,16 @@ module Cosmo
           ok render("jobs/_busy", { jobs: jobs, total: API::Busy.instance.size })
         end
 
-        def _enqueued
+        def _enqueued # rubocop:disable Metrics/AbcSize
           stream_name, stream_names = streams
+          limit = (params["limit"] || API::Stream::LIMIT).to_i
+          page = [params["page"].to_i, 1].max
           stream = API::Stream.new(stream_name)
-          jobs = stream.messages(page: params["page"], limit: params["limit"])
+          total = stream.total
+          jobs = stream.messages(page:, limit:)
+          total_pages = (total.to_f / limit).ceil
 
-          ok render("jobs/_enqueued", { jobs:, total: stream.total, stream_name:, stream_names: })
+          ok render("jobs/_enqueued", { jobs:, total:, stream_name:, stream_names:, page:, limit:, total_pages: })
         end
 
         def _stats

data/lib/cosmo/web/controllers/streams.rb

@@ -40,7 +40,7 @@ module Cosmo
         end
 
         def _table
-          streams = API::Stream.all.map { row_locals(_1) }
+          streams = API::Stream.all.reject { |s| s.name.start_with?("KV_") || s.name.start_with?("_cosmo") }.map { row_locals(_1) }
           ok render("streams/_table", { streams: streams })
         end
 

data/lib/cosmo/web/helpers/application.rb

@@ -73,6 +73,10 @@ module Cosmo
           request_path == path
         end
 
+        def path_prefix?(*values)
+          values.any? { |v| @request.path_info.start_with?(v) }
+        end
+
         def referrer?(path)
           referrer_uri  = URI(@request.referrer)
           referrer_path = referrer_uri.path

data/lib/cosmo/web/views/actions/index.erb

@@ -2,6 +2,6 @@
   <header></header>
 
   <div>
-    <div class="alert alert-info">None</div>
+    <div class="alert alert-info">Coming soon, stay tuned</div>
   </div>
 </section>