cosmonats 0.3.0 → 0.4.0

data/lib/cosmo/job/limit.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Job
+    # Distributed concurrency limiter backed by NATS Key-Value with per-message TTL.
+    #
+    # Each unit of concurrency is a numbered KV slot:
+    #   "{concurrency_key}/0", "{concurrency_key}/1", ..., "{concurrency_key}/{limit-1}"
+    #
+    # Acquiring a slot is a single atomic `set` (CAS with last-revision=0).
+    # Only one worker can win a given slot; losers try the next number.
+    # When a job finishes the slot is deleted; if the worker crashes NATS
+    # expires it automatically via the per-message Nats-TTL header.
+    class Limit
+      BUCKET = "cosmo_jobs_limits"
+
+      def self.instance
+        @instance ||= new
+      end
+
+      def initialize
+        @kv = API::KV.new(BUCKET, allow_msg_ttl: true)
+      end
+
+      # Try to acquire one of the numbered slots for +key+.
+      #
+      # @param key      [String]  concurrency key
+      # @param jid      [String]  stored as the slot value for observability
+      # @param limit    [Integer] number of slots (0 … limit-1)
+      # @param duration [Integer] seconds before the slot is auto-expired by NATS
+      # @return [String, nil] the acquired slot key, or nil when all slots are taken
+      def acquire(key, jid:, limit:, duration:)
+        0.upto(limit - 1) do |i|
+          slot = "#{key}/#{i}"
+          @kv.set(slot, jid, ttl: duration)
+          return slot
+        rescue NATS::KeyValue::KeyWrongLastSequenceError
+          next # slot is live, try the next one
+        end
+        nil # all slots occupied
+      end
+
+      # Release a previously acquired slot.
+      def release(slot)
+        @kv.delete(slot)
+      rescue NATS::Error
+        # best effort — slot TTL will reclaim it if delete fails
+      end
+    end
+  end
+end

data/lib/cosmo/job/processor.rb

@@ -1,11 +1,18 @@
 # frozen_string_literal: true
 
+require "timeout"
+
 module Cosmo
   module Job
     class Processor < ::Cosmo::Processor
       private
 
       def setup
+        # Initialize singletons before starting to process messages
+        API::Busy.instance
+        API::Counter.instance
+        Limit.instance
+
         jobs_config = Config.dig(:consumers, :jobs)
         jobs_config&.each do |stream_name, config|
           next if stream_name == :scheduled # scheduled jobs are handled in schedule_loop
@@ -44,7 +51,7 @@ module Cosmo
         end
       end
 
-      def process(messages, _) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def process(messages, _) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
         message = messages.first
         Logger.debug "received messages #{messages.inspect}"
         data = Utils::Json.parse(message.data)
@@ -61,30 +68,67 @@ module Cosmo
           return
         end
 
+        if worker_class.limits_concurrency?
+          slot = acquire_concurrency_slot(worker_class, message, data)
+          return if slot == false
+        end
+
+        duration = worker_class.default_options[:limit]&.dig(:duration)&.to_i
+
         with_stats(message) do
           sw = stopwatch
           Logger.with(jid: data[:jid])
           Logger.info "start"
           instance = worker_class.new
           instance.jid = data[:jid]
-          instance.perform(*data[:args])
+          if duration
+            Timeout.timeout(duration) { instance.perform(*data[:args]) }
+          else
+            instance.perform(*data[:args])
+          end
           message.ack
           Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "done" }
           true
+        rescue Timeout::Error
+          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[timeout]" }
+          dropped = handle_failure(message, data)
+          false if dropped
         rescue StandardError => e
           Logger.debug e
-          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
+          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[error]" }
           dropped = handle_failure(message, data)
           false if dropped
         rescue Exception # rubocop:disable Lint/RescueException
-          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
+          Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail[exception]" }
           raise
         end
       ensure
+        Limit.instance.release(slot) if slot
         Logger.without(:jid)
         Logger.debug "processed message #{message.inspect}"
       end
 
+      # Tries to acquire a concurrency slot for the job.
+      # Returns the slot key (String) on success, or false if all slots are
+      # taken (message is NAK'd with a delay equal to +duration+ before returning).
+      def acquire_concurrency_slot(worker_class, message, data)
+        options = worker_class.concurrency_options
+        key = worker_class.concurrency_key(data[:args])
+
+        slot = Limit.instance.acquire(key, jid: data[:jid], limit: options[:limit], duration: options[:duration])
+        return slot if slot
+
+        message.nak(delay: options[:duration] * Config::NANO)
+        Logger.debug "concurrency limit reached for #{data[:class]}, re-queueing back #{data[:jid]}"
+        false
+      rescue NATS::Error => e
+        # Unexpected KV failure (e.g. transient NATS error). NAK immediately so
+        # the message is retried rather than stuck in-flight until ack_wait expires.
+        Logger.error e
+        message.nak
+        false
+      end
+
       def handle_failure(message, data) # rubocop:disable Naming/PredicateMethod
         current_attempt = message.metadata.num_delivered
         max_retries = data[:retry].to_i + 1
@@ -93,7 +137,7 @@ module Cosmo
           # NATS will auto-retry with delay (exponential backoff based on current attempt).
           # When max_deliver is reached, NATS stops redelivering the message and marks it as "max deliveries exceeded".
           # The message is effectively abandoned by NATS — it stays in the stream (consuming a slot) but will never be delivered again to that consumer.
-          delay_ns = ((current_attempt**4) + 15) * 1_000_000_000
+          delay_ns = ((current_attempt**4) + 15) * Config::NANO
           message.nak(delay: delay_ns)
           return false
         end

data/lib/cosmo/job.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "cosmo/job/data"
+require "cosmo/job/limit"
 require "cosmo/job/processor"
 
 module Cosmo
@@ -10,8 +11,56 @@ module Cosmo
     end
 
     module ClassMethods
-      def options(stream: nil, retry: nil, dead: nil)
-        default_options.merge!({ stream:, retry:, dead: }.compact)
+      # @option config [Symbol]  :stream  NATS stream to publish to (default: :default)
+      # @option config [Integer] :retry   max delivery attempts before giving up (default: 3)
+      # @option config [Boolean] :dead    move to dead-letter stream after retries exhausted (default: true)
+      # @option config [Hash]    :limit   execution limits:
+      #
+      #   limit: { duration: 30 }
+      #   limit: { duration: 30, concurrency: 3 }
+      #   limit: { duration: 30, concurrency: { to: 3, key: ->(id) { id } } }
+      #
+      # @option config [Integer] :"limit[:duration]"    hard execution timeout in seconds. The job thread is
+      #   killed after this many seconds and counts as a failed attempt (retried with exponential backoff,
+      #   moved to DLQ after retries exhausted).
+      # @option config [Integer, Hash] :"limit[:concurrency]"  caps how many instances run at once across all
+      #   workers. Jobs that cannot acquire a slot are NAK'd with a delay equal to +duration+ so they are not
+      #   re-delivered until the slot is guaranteed free. Requires +duration+.
+      #   Pass an Integer for a class-wide cap, or <tt>{ to: N, key: ->(args) {} }</tt> to scope per key.
+      def options(**config)
+        if config[:limit] && config.dig(:limit, :concurrency) && !config.dig(:limit, :duration).to_i.positive?
+          raise ArgumentError, "limit: duration is required when concurrency is set"
+        end
+
+        default_options.merge!(config)
+      end
+      alias cosmo_options options
+
+      def limits_concurrency?
+        !!concurrency_options
+      end
+
+      # Returns a normalized concurrency config hash, or +nil+ when not configured.
+      # Always contains +:limit+, +:key+, and +:duration+.
+      def concurrency_options
+        value = default_options.dig(:limit, :concurrency)
+        duration = default_options.dig(:limit, :duration).to_i
+        return unless value
+
+        case value
+        when Integer then { limit: value, key: nil, duration: duration }
+        when Hash    then { limit: value.fetch(:to), key: value[:key], duration: duration }
+        end
+      end
+
+      # Derive the fully-scoped concurrency key for a given args array.
+      def concurrency_key(args)
+        config = concurrency_options
+        return unless config
+
+        base = Utils::String.underscore(name)
+        suffix = config[:key]&.call(*args)
+        suffix ? "#{base}/#{suffix}" : base
       end
 
       def perform(*args, async: true, **options)

data/lib/cosmo/processor.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Cosmo
-  class Processor # rubocop:disable Metrics/ClassLength
+  class Processor
     STREAM_PAUSED_RECHECK_TTL = 5.0 # Seconds a stream's paused state is cached before re-checking (override via COSMO_STREAM_PAUSED_RECHECK_TTL)
     STREAMS_PAUSED_IDLE_SLEEP = 1.0 # Seconds to sleep when every stream is paused, preventing a tight CPU spin (override via COSMO_STREAMS_PAUSED_IDLE_SLEEP)
     STREAM_EMPTY_BACKOFF_MAX = 5.0  # Max seconds to sleep between empty fetches (override via COSMO_STREAM_EMPTY_BACKOFF_MAX)

data/lib/cosmo/railtie.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Cosmo
+  # Rails Railtie — loaded automatically when cosmonats is required inside a
+  # Rails application. Ensures the ActiveJob adapter constant is registered
+  # before Rails tries to resolve it and autoloads +config/cosmo.yml+ when
+  # the file is present.
+  class Railtie < ::Rails::Railtie
+    # Make Cosmo::ActiveJobAdapter::Adapter available under the conventional
+    # ActiveJob namespace so :cosmonats resolves without any extra requires.
+    initializer "cosmo.active_job_adapter", before: :run_prepare_callbacks do
+      require "cosmo/active_job"
+    end
+
+    # Autoload config/cosmo.yml when it exists and no config has been loaded yet.
+    initializer "cosmo.load_config", after: "cosmo.active_job_adapter" do |app|
+      config_path = app.root.join("config", "cosmo.yml")
+      Config.load(config_path.to_s) if config_path.exist? && Config.instance.none?
+    end
+  end
+end

data/lib/cosmo/stream/processor.rb

@@ -17,7 +17,7 @@ module Cosmo
         @configs.each { @consumers << subscribe(nil, _1) }
       end
 
-      def process(messages, processor) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def process(messages, processor) # rubocop:disable Metrics/AbcSize
         metadata = messages.last.metadata
         serializer = processor.class.default_options.dig(:publisher, :serializer)
         messages = messages.map { Message.new(_1, serializer:) }
@@ -49,7 +49,7 @@ module Cosmo
       end
 
       def dynamic_config
-        Config.internal[:streams].map { _1.default_options.merge(class: _1) }
+        Config.internal[:streams]&.map { _1.default_options.merge(class: _1) }.to_a
       end
 
       def subscribe(_stream_name, config)

data/lib/cosmo/stream.rb

@@ -12,10 +12,11 @@ module Cosmo
     end
 
     module ClassMethods
-      def options(stream: nil, consumer_name: nil, batch_size: nil, fetch_timeout: nil, start_position: nil, consumer: nil, publisher: nil) # rubocop:disable Metrics/ParameterLists
+      def options(stream: nil, consumer_name: nil, batch_size: nil, fetch_timeout: nil, start_position: nil, consumer: nil, publisher: nil)
         register
         default_options.merge!({ stream:, consumer_name:, batch_size:, fetch_timeout:, start_position:, consumer:, publisher: }.compact)
       end
+      alias cosmo_options options
 
       def publish(data, subject: nil, **options)
         stream = default_options[:stream]

data/lib/cosmo/utils/hash.rb

@@ -23,6 +23,19 @@ module Cosmo
         end
       end
 
+      def stringify_keys(obj)
+        case obj
+        when ::Hash
+          obj.each_with_object({}) do |(key, value), result|
+            result[key.to_s] = stringify_keys(value)
+          end
+        when ::Array
+          obj.map { |v| stringify_keys(v) }
+        else
+          obj
+        end
+      end
+
       # deep set
       def set(hash, *keys, value)
         last_key = keys.pop

data/lib/cosmo/utils/overrides.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 Cosmo::Utils::Warnings.silence do
-  members = NATS::JetStream::API::StreamConfig.members + [:allow_msg_counter]
+  members = NATS::JetStream::API::StreamConfig.members + %i[allow_msg_counter allow_msg_schedules]
   NATS::JetStream::API::StreamConfig = Struct.new(*members, keyword_init: true) do
     def initialize(opts = {})
       rem = opts.keys - members

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/cosmo/web/assets/app.css

@@ -292,6 +292,34 @@ section > header {
   z-index: auto;
 }
 
+/* ── Section tabs ───────────────────────────────────────────────────────── */
+.tabs {
+  display: flex;
+  gap: 0;
+  border-bottom: 2px solid var(--color-border);
+  margin-bottom: var(--space-3x);
+}
+
+.tabs a {
+  padding: var(--space) var(--space-3x);
+  text-decoration: none;
+  color: var(--color-text-light);
+  border-bottom: 2px solid transparent;
+  margin-bottom: -2px;
+  font-weight: 500;
+  transition: color 0.15s, border-color 0.15s;
+}
+
+.tabs a:hover {
+  color: var(--color-text);
+  border-bottom-color: var(--color-border);
+}
+
+.tabs a.active {
+  color: var(--color-primary);
+  border-bottom-color: var(--color-primary);
+}
+
 section .nav {
   display: flex;
   gap: var(--space);
@@ -448,6 +476,20 @@ time {
 .stream-link { color: var(--color-primary); font-weight: 600; }
 .stream-link:hover { text-decoration: underline; }
 
+/* ── Pagination ─────────────────────────────────────────────────────────── */
+.pagination {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: var(--space);
+  padding: var(--space-2x) 0 var(--space);
+}
+.btn-disabled {
+  opacity: 0.4;
+  cursor: default;
+  pointer-events: none;
+}
+
 /* ── Actions ───────────────────────────────────────────────────────────── */
 .actions-form { display: flex; flex-direction: column; gap: var(--space-1-2); }
 

data/lib/cosmo/web/controllers/crons.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "cosmo/web/controllers/application"
+
+module Cosmo
+  class Web
+    module Controllers
+      class Crons < Application
+        def index
+          content_for :title, "Crons"
+          ok render("crons/index", layout: true)
+        end
+
+        def _table
+          ok render("crons/_table", { schedules: cron.all })
+        end
+
+        # Dispatch the job immediately, bypassing the schedule timer.
+        # Expects params["subject"] = the schedule subject stored in NATS.
+        def run_now
+          subject = Rack::Utils.unescape(params["subject"].to_s)
+          cron.run_now!(subject)
+          ok
+        end
+
+        # Purge the schedule from NATS so it stops firing.
+        def delete
+          subject = Rack::Utils.unescape(params["subject"].to_s)
+          cron.delete!(subject)
+          _table
+        end
+
+        private
+
+        def cron
+          @cron ||= API::Cron.instance
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/web/controllers/jobs.rb

@@ -72,12 +72,16 @@ module Cosmo
           ok render("jobs/_busy", { jobs: jobs, total: API::Busy.instance.size })
         end
 
-        def _enqueued
+        def _enqueued # rubocop:disable Metrics/AbcSize
           stream_name, stream_names = streams
+          limit = (params["limit"] || API::Stream::LIMIT).to_i
+          page = [params["page"].to_i, 1].max
           stream = API::Stream.new(stream_name)
-          jobs = stream.messages(page: params["page"], limit: params["limit"])
+          total = stream.total
+          jobs = stream.messages(page:, limit:)
+          total_pages = (total.to_f / limit).ceil
 
-          ok render("jobs/_enqueued", { jobs:, total: stream.total, stream_name:, stream_names: })
+          ok render("jobs/_enqueued", { jobs:, total:, stream_name:, stream_names:, page:, limit:, total_pages: })
         end
 
         def _stats

data/lib/cosmo/web/controllers/streams.rb

@@ -40,7 +40,7 @@ module Cosmo
         end
 
         def _table
-          streams = API::Stream.all.map { row_locals(_1) }
+          streams = API::Stream.all.reject { |s| s.name.start_with?("KV_") || s.name.start_with?("_cosmo") }.map { row_locals(_1) }
           ok render("streams/_table", { streams: streams })
         end
 

data/lib/cosmo/web/helpers/application.rb

@@ -73,6 +73,10 @@ module Cosmo
           request_path == path
         end
 
+        def path_prefix?(*values)
+          values.any? { |v| @request.path_info.start_with?(v) }
+        end
+
         def referrer?(path)
           referrer_uri  = URI(@request.referrer)
           referrer_path = referrer_uri.path

data/lib/cosmo/web/views/actions/index.erb

@@ -2,6 +2,6 @@
   <header></header>
 
   <div>
-    <div class="alert alert-info">None</div>
+    <div class="alert alert-info">Coming soon, stay tuned</div>
   </div>
 </section>

data/lib/cosmo/web/views/crons/_table.erb

@@ -0,0 +1,58 @@
+<% if @schedules.empty? -%>
+  <div class="alert alert-success">
+    No cron schedules deployed in NATS.<br>
+    Add a <code>cron:</code> section to <code>cosmo.yml</code> and run
+    <code>cosmo --setup</code> to publish them.
+  </div>
+<% else -%>
+  <div class="table-container">
+    <table>
+      <thead>
+        <tr>
+          <th>Job</th>
+          <th>Expression</th>
+          <th>Timezone</th>
+          <th>Args</th>
+          <th>Stream</th>
+          <th>Actions</th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @schedules.each do |s| -%>
+        <tr id="cron-row-<%= h(s[:registry_key].to_s.gsub("/", "-")) %>">
+          <td>
+            <div class="job-class"><%= h(s[:class].to_s) %></div>
+            <% if s[:name] -%>
+              <code class="text-muted" style="font-size:0.8em;"><%= h(s[:name].to_s) %></code>
+            <% end -%>
+            <div style="font-size:0.75em; color:#888; margin-top:2px;">
+              <code><%= h(s[:schedule_subject].to_s) %></code>
+            </div>
+          </td>
+          <td><code class="subject-tag"><%= h(s[:schedule].to_s) %></code></td>
+          <td><%= h(s[:timezone] || "UTC") %></td>
+          <td>
+            <% if s[:args]&.any? -%>
+              <code><%= h(s[:args].inspect) %></code>
+            <% else -%>
+              <span class="text-muted">&mdash;</span>
+            <% end -%>
+          </td>
+          <td><code><%= h(s[:stream].to_s) %></code></td>
+          <td style="white-space:nowrap;">
+            <a hx-post="<%= url_for('/crons/run', { subject: u(s[:schedule_subject].to_s) }) %>"
+               class="btn btn-success btn-sm"
+               title="Dispatch this job now, bypassing the schedule timer">&#x25BA; Run Now</a>
+            <a hx-delete="<%= url_for('/crons/delete', { subject: u(s[:schedule_subject].to_s) }) %>"
+               hx-target="#crons-table"
+               hx-swap="innerHTML"
+               hx-confirm="Delete '<%= h(s[:schedule_subject].to_s) %>'? NATS will stop firing it."
+               class="btn btn-danger btn-sm"
+               title="Purge schedule from NATS">&#x1F5D1; Delete</a>
+          </td>
+        </tr>
+        <% end -%>
+      </tbody>
+    </table>
+  </div>
+<% end -%>

data/lib/cosmo/web/views/crons/index.erb

@@ -0,0 +1,10 @@
+<section>
+  <header><%= render('jobs/_tabs') %></header>
+
+  <div id="crons-table"
+       hx-get="<%= url_for('/crons/_table') %>"
+       hx-trigger="load, every 10s"
+       hx-swap="innerHTML">
+    <div class="alert alert-info">Loading cron schedules&hellip;</div>
+  </div>
+</section>

data/lib/cosmo/web/views/jobs/_busy.erb

@@ -1,50 +1,55 @@
-<div class="pending">
-  <span></span>
-  <span class="text-muted"><%= @total %> job(s) running</span>
-</div>
-
-<% if @jobs.empty? -%>
-  <div class="alert alert-success">No jobs are currently running</div>
-<% else -%>
-  <div class="table-container">
-    <table>
-      <thead>
-        <tr>
-          <th>Job</th>
-          <th>JID[Args]</th>
-          <th>Stream</th>
-          <th>Worker</th>
-          <th>Running for</th>
-        </tr>
-      </thead>
-      <tbody>
-        <% @jobs.each do |job| -%>
-        <tr>
-          <td>
-            <div class="job-class">
-              <%= h(job.dig(:data, :class).to_s) %>
-            </div>
-          </td>
-          <td>
-            <div class="job-id">
-              <details>
-                <summary>
-                  <code><%= h(job.dig(:data, :jid)) %></code>
-                </summary>
-                <code><%= h(JSON.pretty_generate(job.dig(:data, :args))) %></code>
-              </details>
-            </div>
-          </td>
-          <td>
-            <code><%= h(job[:stream].to_s) %></code>
-          </td>
-          <td><code style="font-size: var(--font-size-small);"><%= h(job[:worker].to_s) %></code></td>
-          <td>
-            <%= elapsed(job[:started_at]) %>
-          </td>
-        </tr>
-        <% end -%>
-      </tbody>
-    </table>
+<div id="busy-poller"
+     hx-get="<%= url_for('/jobs/_busy') %>"
+     hx-trigger="every 5s"
+     hx-swap="outerHTML">
+  <div class="pending">
+    <span></span>
+    <span class="text-muted"><%= @total %> job(s) running</span>
   </div>
-<% end -%>
+
+  <% if @jobs.empty? -%>
+    <div class="alert alert-success">No jobs are currently running</div>
+  <% else -%>
+    <div class="table-container">
+      <table>
+        <thead>
+          <tr>
+            <th>Job</th>
+            <th>JID[Args]</th>
+            <th>Stream</th>
+            <th>Worker</th>
+            <th>Running for</th>
+          </tr>
+        </thead>
+        <tbody>
+          <% @jobs.each do |job| -%>
+          <tr>
+            <td>
+              <div class="job-class">
+                <%= h(job.dig(:data, :class).to_s) %>
+              </div>
+            </td>
+            <td>
+              <div class="job-id">
+                <details>
+                  <summary>
+                    <code><%= h(job.dig(:data, :jid)) %></code>
+                  </summary>
+                  <code><%= h(JSON.pretty_generate(job.dig(:data, :args))) %></code>
+                </details>
+              </div>
+            </td>
+            <td>
+              <code><%= h(job[:stream].to_s) %></code>
+            </td>
+            <td><code style="font-size: var(--font-size-small);"><%= h(job[:worker].to_s) %></code></td>
+            <td>
+              <%= elapsed(job[:started_at]) %>
+            </td>
+          </tr>
+          <% end -%>
+        </tbody>
+      </table>
+    </div>
+  <% end -%>
+</div>