cosmonats 0.6.0 → 0.7.0

data/lib/cosmo/processor.rb

@@ -5,20 +5,29 @@ module Cosmo
     STREAM_PAUSED_RECHECK_TTL = "5s" # How long a stream's paused state is cached before re-checking (override via COSMO_STREAM_PAUSED_RECHECK_TTL)
     STREAMS_PAUSED_IDLE_SLEEP = "1s" # How long to sleep when every stream is paused, preventing a tight CPU spin (override via COSMO_STREAMS_PAUSED_IDLE_SLEEP)
     STREAM_EMPTY_BACKOFF_MAX = "5s"  # Max sleep between empty fetches (override via COSMO_STREAM_EMPTY_BACKOFF_MAX)
+    QUIET_SLEEP = "5s"               # How long to sleep between checks while in quiet mode (override via COSMO_QUIET_IDLE_SLEEP)
 
     def self.run(...)
       new(...).tap(&:run)
     end
 
+    # Checks the command-line options this processor understands, before anything is booted or connected.
+    #
+    # @param options [Hash]
+    # @return [void]
+    def self.validate_options!(_options); end
+
     attr_reader :consumers
 
-    def initialize(pool, running, options)
+    def initialize(pool, running, options, quiet:)
       @pool = pool
       @running = running
+      @quiet = quiet
       @options = options
       @threads = []
       @consumers = []
       @cache = Utils::TTLCache.new
+      @busy = Concurrent::AtomicFixnum.new
     end
 
     def run
@@ -29,6 +38,13 @@ module Cosmo
       run_loop
     end
 
+    # Threads processing fetched messages right now; threads waiting on a fetch don't count.
+    #
+    # @return [Integer]
+    def busy
+      @busy.value
+    end
+
     def stop(timeout = Config[:timeout])
       @running.make_false
       @pool.shutdown
@@ -52,6 +68,11 @@ module Cosmo
       while running?
         break if shutdown
 
+        if quiet?
+          sleep(Utils::Duration.parse(ENV.fetch("COSMO_QUIET_IDLE_SLEEP", QUIET_SLEEP)))
+          next
+        end
+
         all_empty = true  # every stream is empty
         all_paused = true # every stream is paused
         consumers.each do |(subscription, config, processor)| # rubocop:disable Metrics/BlockLength
@@ -80,15 +101,18 @@ module Cosmo
 
               timeout = fetch_timeout(config)
               Logger.trace "fetching #{fetch_subjects(config).inspect}, timeout=#{timeout}"
+              started_at = Time.now
               messages = fetch(subscription, batch_size: config[:batch_size], timeout:)
               Logger.trace "fetched (#{messages&.size.to_i}) messages"
               if messages&.any?
                 consumer_state.delete(stream_name)
-                process(messages, processor)
+                track_busy { process(messages, processor) }
               else
                 max_backoff = Utils::Duration.parse(ENV.fetch("COSMO_STREAM_EMPTY_BACKOFF_MAX", STREAM_EMPTY_BACKOFF_MAX))
-                consumer_state.compute(stream_name) do |current|
-                  count = (current&.first || 0) + 1
+                consumer_state.compute(stream_name) do |(count, wake_at)|
+                  next [count, wake_at] if wake_at && wake_at > started_at
+
+                  count = count.to_i + 1
                   backoff = [timeout * (2**(count - 1)), max_backoff].min
                   [count, Time.now + backoff]
                 end
@@ -129,10 +153,21 @@ module Cosmo
       raise NotImplementedError
     end
 
+    def track_busy
+      @busy.increment
+      yield
+    ensure
+      @busy.decrement
+    end
+
     def running?
       @running.true?
     end
 
+    def quiet?
+      @quiet.true?
+    end
+
     def scheduler?
       false
     end

data/lib/cosmo/sentry/job_processor_middleware.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+# rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize
 module Cosmo
   module Sentry
     module JobProcessorMiddleware
@@ -63,4 +63,3 @@ module Cosmo
     end
   end
 end
-# rubocop:enable Metrics/MethodLength, Metrics/AbcSize

data/lib/cosmo/stream/message.rb

@@ -7,8 +7,9 @@ module Cosmo
     class Message
       extend Forwardable
 
-      delegate %i[subject reply header metadata ack nack term in_progress] => :@msg
+      delegate %i[subject reply header metadata ack nak term in_progress] => :@msg
       delegate %i[timestamp num_delivered num_pending] => :metadata
+      alias nack nak
 
       def initialize(msg, serializer: nil)
         @msg = msg

data/lib/cosmo/stream/processor.rb

@@ -3,6 +3,11 @@
 module Cosmo
   module Stream
     class Processor < ::Cosmo::Processor
+      # @return [Array<String>] names of the stream classes being processed
+      def subscriptions
+        @consumers.map { |(_, config, _)| config[:class].name }
+      end
+
       private
 
       def setup

data/lib/cosmo/utils/json.rb

@@ -8,7 +8,7 @@ module Cosmo
       module_function
 
       def parse(value, default: nil, symbolize_names: true, **options)
-        JSON.parse(value, options.merge(symbolize_names:))
+        JSON.parse(value, **options, symbolize_names:)
       rescue TypeError, JSON::ParserError
         default
       end

data/lib/cosmo/utils/overrides.rb

@@ -1,20 +1,24 @@
 # frozen_string_literal: true
 
 Cosmo::Utils::Warnings.silence do
-  members = NATS::JetStream::API::StreamConfig.members + %i[allow_msg_counter allow_msg_schedules]
-  NATS::JetStream::API::StreamConfig = Struct.new(*members, keyword_init: true) do
-    def initialize(opts = {})
-      rem = opts.keys - members
-      opts.delete_if { |k| rem.include?(k) }
-      super
+  unless NATS::JetStream::API::StreamConfig.members.include?(:allow_msg_schedules)
+    members = NATS::JetStream::API::StreamConfig.members + %i[allow_msg_counter allow_msg_schedules]
+    NATS::JetStream::API::StreamConfig = Struct.new(*members, keyword_init: true) do
+      def initialize(opts = {})
+        rem = opts.keys - members
+        opts.delete_if { |k| rem.include?(k) }
+        super
+      end
     end
   end
 
-  members = NATS::JetStream::PubAck.members + [:val]
-  NATS::JetStream::PubAck = Struct.new(*members, keyword_init: true)
+  unless NATS::JetStream::PubAck.members.include?(:val)
+    members = NATS::JetStream::PubAck.members + [:val]
+    NATS::JetStream::PubAck = Struct.new(*members, keyword_init: true)
+  end
 end
 
-# Upstream bug in nats-pure 2.5.0 (https://github.com/nats-io/nats.rb):
+# Upstream bug in nats-pure 2.5.0 and 2.6.0 (https://github.com/nats-io/nats-pure.rb):
 # NATS::JetStream::PullSubscription#fetch only assigns its `start_time` local in the
 # `batch > 1` branch, but the final "did we time out" check outside the case statement
 # reads it unconditionally. For `batch == 1` (what Cosmo::Processor always uses),
@@ -44,7 +48,9 @@ end
 #
 # Vendored copy of nats-pure's PullSubscription#fetch, kept as close to
 # upstream as possible (two one-line fixes, see comments above), so it's easy to diff against the
-# next nats-pure release and drop once fixed there.
+# next nats-pure release and drop once fixed there. Fixed on nats-pure main by the fetch rewrite
+# (#181), recognized by its private #take_pending, in which case this copy is not loaded.
+return if NATS::JetStream.const_get(:PullSubscription).private_method_defined?(:take_pending)
 
 # rubocop:disable all
 module NATS

data/lib/cosmo/utils/thread_pool.rb

@@ -16,11 +16,16 @@ module Cosmo
 
       def initialize(concurrency)
         @mutex = Thread::Mutex.new
+        @concurrency = concurrency
         @available = concurrency
         @cond = ConditionVariable.new
         @pool = Concurrent::FixedThreadPool.new(concurrency)
       end
 
+      def wait_idle
+        @mutex.synchronize { @cond.wait(@mutex) until @available == @concurrency }
+      end
+
       def post
         @mutex.synchronize do
           @cond.wait(@mutex) while @available <= 0

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/lib/cosmo/web/assets/app.css

@@ -163,6 +163,14 @@ button:hover, .btn:hover {
   );
 }
 
+.btn-sm {
+  box-shadow: none;
+  font-size: var(--font-size-small);
+  font-weight: 600;
+  line-height: var(--space-2x);
+  padding: var(--space-1-2) var(--space);
+}
+
 /* ── Badges ────────────────────────────────────────────────────────────── */
 .badge {
   border-radius: var(--space-1-2);
@@ -343,6 +351,12 @@ section .pending {
   align-items: center;
 }
 
+section .pending .poll-status {
+  align-items: center;
+  display: inline-flex;
+  gap: var(--space);
+}
+
 /* ── Stat cards ────────────────────────────────────────────────────────── */
 .cards-container {
   display: grid;
@@ -479,6 +493,13 @@ time {
   font-size: var(--font-size-small);
   word-wrap: break-word;
 }
+.error-message details code {
+  max-width: 600px;
+  max-height: 320px;
+  overflow: auto;
+  white-space: pre;
+  font-size: var(--font-size-small);
+}
 
 /* ── Links ─────────────────────────────────────────────────────────────── */
 .stream-link { color: var(--color-primary); font-weight: 600; }

data/lib/cosmo/web/controllers/jobs.rb

@@ -54,6 +54,13 @@ module Cosmo
           ok
         end
 
+        def delete_enqueued
+          seq = path.split("/").last.to_i
+          stream_name, = streams
+          API::Stream.new(stream_name).delete(seq)
+          _enqueued
+        end
+
         def _scheduled
           stream = API::Stream.new("scheduled")
           jobs = stream.messages(page: params["page"], limit: params["limit"])
@@ -67,9 +74,19 @@ module Cosmo
         end
 
         def _busy
-          limit = (limit || 25).to_i
-          jobs  = API::Busy.instance.list(limit:)
-          ok render("jobs/_busy", { jobs: jobs, total: API::Busy.instance.size })
+          busy = API::Stats::Busy.instance
+          total = busy.size
+          page, limit, total_pages = paginate(total, API::Stats::Busy::LIMIT)
+          polling = params["poll"].to_s != "0"
+          ok render("jobs/_busy", { jobs: busy.list(page:, limit:), total:, page:, limit:, total_pages:, polling: })
+        end
+
+        def _processes
+          processes = API::Stats::Processes.instance
+          total = processes.size
+          page, limit, total_pages = paginate(total, API::Stats::Processes::LIMIT)
+          polling = params["poll"].to_s != "0"
+          ok render("jobs/_processes", { processes: processes.list(page:, limit:), total:, page:, limit:, total_pages:, polling: })
         end
 
         def _enqueued # rubocop:disable Metrics/AbcSize
@@ -80,8 +97,9 @@ module Cosmo
           unless stream_name.to_s.empty?
             stream = API::Stream.new(stream_name)
             total = stream.total
-            jobs = stream.messages(page:, limit:)
             total_pages = (total.to_f / limit).ceil
+            page = page.clamp(1, [total_pages, 1].max)
+            jobs = stream.messages(page:, limit:)
           end
 
           ok render("jobs/_enqueued", { jobs:, total:, stream_name:, stream_names:, page:, limit:, total_pages: })
@@ -93,6 +111,12 @@ module Cosmo
 
         private
 
+        def paginate(total, default_limit)
+          limit = (params["limit"] || default_limit).to_i
+          total_pages = (total.to_f / limit).ceil
+          [params["page"].to_i.clamp(1, [total_pages, 1].max), limit, total_pages]
+        end
+
         def streams
           stream_names = API::Stream.jobs.map(&:name)
           stream_name = params.fetch("stream_name", stream_names.first)

data/lib/cosmo/web/helpers/application.rb

@@ -8,6 +8,13 @@ module Cosmo
       module Application
         include Renderer
 
+        # Seconds between htmx auto-refreshes. Override with COSMO_WEB_POLL_INTERVAL.
+        POLL_INTERVAL = 5
+
+        def poll_interval
+          @poll_interval ||= ENV.fetch("COSMO_WEB_POLL_INTERVAL", POLL_INTERVAL).to_i
+        end
+
         def render(template, locals = nil)
           defaults = { request: @request }
           locals = Hash(locals).merge(defaults)
@@ -42,8 +49,10 @@ module Cosmo
             "#{elapsed}s"
           elsif elapsed < 3600
             "#{elapsed / 60}m #{elapsed % 60}s"
-          else
+          elsif elapsed < 86_400
             "#{elapsed / 3600}h #{(elapsed % 3600) / 60}m"
+          else
+            "#{elapsed / 86_400}d #{(elapsed % 86_400) / 3600}h"
           end
         end
 

data/lib/cosmo/web/views/batches/index.erb

@@ -3,7 +3,7 @@
 
   <div id="batches-table"
        hx-get="<%= url_for('/batches/_table') %>"
-       hx-trigger="load, every 5s"
+       hx-trigger="load, every <%= poll_interval %>s"
        hx-swap="innerHTML">
     <div class="alert alert-info">Loading batches&hellip;</div>
   </div>

data/lib/cosmo/web/views/jobs/_busy.erb

@@ -1,10 +1,29 @@
+<% nav = @polling ? { limit: @limit } : { limit: @limit, poll: 0 } -%>
 <div id="busy-poller"
-     hx-get="<%= url_for('/jobs/_busy') %>"
-     hx-trigger="every 5s"
-     hx-swap="outerHTML">
+     <% if @polling -%>
+     hx-get="<%= url_for('/jobs/_busy', page: @page, limit: @limit) %>"
+     hx-trigger="every <%= poll_interval %>s"
+     hx-swap="outerHTML"
+     <% end -%>
+     >
   <div class="pending">
     <span></span>
-    <span class="text-muted"><%= @total %> job(s) running</span>
+    <span class="poll-status">
+      <span class="text-muted"><%= @total %> job(s) running</span>
+      <% if @polling -%>
+        <a class="btn btn-sm"
+           hx-get="<%= url_for('/jobs/_busy', nav.merge(page: @page, poll: 0)) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner">&#9209; Stop Polling</a>
+      <% else -%>
+        <a class="btn btn-sm btn-primary"
+           hx-get="<%= url_for('/jobs/_busy', page: @page, limit: @limit) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner">&#x25B6; Live Poll</a>
+      <% end -%>
+    </span>
   </div>
 
   <% if @jobs.empty? -%>
@@ -18,6 +37,7 @@
             <th>JID[Args]</th>
             <th>Stream</th>
             <th>Worker</th>
+            <th>Attempt</th>
             <th>Running for</th>
           </tr>
         </thead>
@@ -43,6 +63,14 @@
               <code><%= h(job[:stream].to_s) %></code>
             </td>
             <td><code style="font-size: var(--font-size-small);"><%= h(job[:worker].to_s) %></code></td>
+            <td>
+              <% delivery = job[:delivery].to_i -%>
+              <% if delivery > 1 -%>
+                <span class="badge <%= delivery >= 5 ? 'badge-danger' : 'badge-warning' %>"><%= delivery %></span>
+              <% else -%>
+                <span class="text-muted"><%= delivery.zero? ? '&mdash;' : delivery %></span>
+              <% end -%>
+            </td>
             <td>
               <%= elapsed(job[:started_at]) %>
             </td>
@@ -52,4 +80,42 @@
       </table>
     </div>
   <% end -%>
+
+  <% if @total_pages.to_i > 1 -%>
+    <div class="pagination">
+      <% if @page > 1 -%>
+        <a hx-get="<%= url_for('/jobs/_busy', nav.merge(page: @page - 1)) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner"
+           class="btn">&#x2190; Prev</a>
+      <% else -%>
+        <span class="btn btn-disabled">&#x2190; Prev</span>
+      <% end -%>
+
+      <% pages(@page, @total_pages).each do |p| -%>
+        <% if p == :gap -%>
+          <span class="pagination-gap">&hellip;</span>
+        <% elsif p == @page -%>
+          <span class="btn btn-primary btn-disabled"><%= p %></span>
+        <% else -%>
+          <a hx-get="<%= url_for('/jobs/_busy', nav.merge(page: p)) %>"
+             hx-target="#busy-poller"
+             hx-swap="outerHTML"
+             hx-indicator="#global-spinner"
+             class="btn"><%= p %></a>
+        <% end -%>
+      <% end -%>
+
+      <% if @page < @total_pages -%>
+        <a hx-get="<%= url_for('/jobs/_busy', nav.merge(page: @page + 1)) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner"
+           class="btn">Next &#x2192;</a>
+      <% else -%>
+        <span class="btn btn-disabled">Next &#x2192;</span>
+      <% end -%>
+    </div>
+  <% end -%>
 </div>

data/lib/cosmo/web/views/jobs/_dead.erb

@@ -1,7 +1,4 @@
-<div id="dead-poller"
-     hx-get="<%= url_for('/jobs/_dead') %>"
-     hx-trigger="every 5s"
-     hx-swap="outerHTML">
+<div id="dead-jobs">
   <% if @jobs.empty? -%>
     <div class="alert alert-success">No dead jobs found</div>
   <% else -%>
@@ -35,31 +32,47 @@
                 </details>
               </div>
             </td>
-            <td>
-              <code><%= h(job.stream) %></code>
+            <td style="white-space: nowrap;">
+              <% if job.x_stream -%><code><%= h(job.x_stream) %></code><% else -%><span class="text-muted">&mdash;</span><% end -%>
             </td>
-            <td style="max-width: 400px;">
-              <% if job.data[:error] -%>
+            <td style="min-width: 420px;">
+              <% if job.error? -%>
               <div class="error-message">
-                <strong><%= h(job.data[:error].to_s.split(":").first || "Error") %></strong>
-                <p><%= h(job.data[:error]) %></p>
+                <strong><%= h(job.error_class.to_s.empty? ? "Error" : job.error_class) %></strong>
+                <p><%= h(job.error_message) %></p>
+                <% if job.error_backtrace -%>
+                <details>
+                  <summary>Backtrace</summary>
+                  <code><%= h(job.error_backtrace_lines.join("\n")) %></code>
+                </details>
+                <% end -%>
+              </div>
+              <% else -%>
+              <div class="error-message">
+                <strong>Unknown</strong>
+                <p>No error recorded for this job</p>
               </div>
               <% end -%>
             </td>
             <td style="white-space: nowrap;">
-              <time><%= h(job.timestamp) %></time>
+              <% if (failed_at = job.stored_at) -%>
+              <time datetime="<%= h(job.timestamp) %>" title="<%= h(job.timestamp) %>"><%= h(format_timestamp(failed_at)) %></time>
+              <div class="text-muted" style="font-size: var(--font-size-small);"><%= elapsed(failed_at) %> ago</div>
+              <% else -%>
+              <span class="text-muted">&mdash;</span>
+              <% end -%>
             </td>
-            <td>
+            <td style="white-space: nowrap;">
               <a hx-patch="<%= url_for("/jobs/retry/#{job.seq}") %>"
                  hx-target="#dead-row-<%= job.seq %>"
                  hx-swap="innerHTML"
-                 class="btn btn-primary"
-                 style="width: 100%; text-align: center; margin-bottom: 3px">&#x21BB; Retry</a>
+                 title="Retry"
+                 class="btn btn-primary btn-sm">&#x21BB;</a>
               <a hx-delete="<%= url_for("/jobs/delete/#{job.seq}") %>"
                  hx-target="#dead-row-<%= job.seq %>"
                  hx-swap="innerHTML"
-                 class="btn btn-danger"
-                 style="width: 100%; text-align: center; margin-bottom: 3px">&#x2715; Delete</a>
+                 title="Delete"
+                 class="btn btn-danger btn-sm">&#x2715;</a>
             </td>
           </tr>
           <% end -%>

data/lib/cosmo/web/views/jobs/_enqueued.erb

@@ -30,6 +30,7 @@
             <th>JID[Args]</th>
             <th>Stream</th>
             <th>Seq</th>
+            <th>Actions</th>
           </tr>
         </thead>
         <tbody>
@@ -56,6 +57,15 @@
             <td>
               <code><%= job.seq %></code>
             </td>
+            <td>
+              <a hx-delete="<%= url_for("/jobs/enqueued/#{job.seq}", { stream_name: @stream_name, page: @page, limit: @limit }) %>"
+                 hx-target="#enqueued-poller"
+                 hx-swap="outerHTML"
+                 hx-indicator="#global-spinner"
+                 hx-confirm="Remove message #<%= job.seq %> from <%= h(@stream_name) %>?"
+                 title="Remove message"
+                 class="btn btn-danger btn-sm">&#x2715;</a>
+            </td>
           </tr>
           <% end -%>
         </tbody>