cosmonats 0.5.1 → 0.7.0

data/lib/cosmo/utils/overrides.rb

@@ -1,15 +1,206 @@
 # frozen_string_literal: true
 
 Cosmo::Utils::Warnings.silence do
-  members = NATS::JetStream::API::StreamConfig.members + %i[allow_msg_counter allow_msg_schedules]
-  NATS::JetStream::API::StreamConfig = Struct.new(*members, keyword_init: true) do
-    def initialize(opts = {})
-      rem = opts.keys - members
-      opts.delete_if { |k| rem.include?(k) }
-      super
+  unless NATS::JetStream::API::StreamConfig.members.include?(:allow_msg_schedules)
+    members = NATS::JetStream::API::StreamConfig.members + %i[allow_msg_counter allow_msg_schedules]
+    NATS::JetStream::API::StreamConfig = Struct.new(*members, keyword_init: true) do
+      def initialize(opts = {})
+        rem = opts.keys - members
+        opts.delete_if { |k| rem.include?(k) }
+        super
+      end
     end
   end
 
-  members = NATS::JetStream::PubAck.members + [:val]
-  NATS::JetStream::PubAck = Struct.new(*members, keyword_init: true)
+  unless NATS::JetStream::PubAck.members.include?(:val)
+    members = NATS::JetStream::PubAck.members + [:val]
+    NATS::JetStream::PubAck = Struct.new(*members, keyword_init: true)
+  end
+end
+
+# Upstream bug in nats-pure 2.5.0 and 2.6.0 (https://github.com/nats-io/nats-pure.rb):
+# NATS::JetStream::PullSubscription#fetch only assigns its `start_time` local in the
+# `batch > 1` branch, but the final "did we time out" check outside the case statement
+# reads it unconditionally. For `batch == 1` (what Cosmo::Processor always uses),
+# `start_time` is nil there -- normally masked because the `batch == 1` branch has its
+# own timeout check (using a properly-set `t`) that raises first.
+#
+# It only surfaces once two threads call #fetch concurrently on the *same* subscription:
+# @pending_queue/wait_for_msgs_cond are shared per-subscription state, and #dispatch's
+# `wait_for_msgs_cond.signal` wakes exactly one arbitrary waiter -- not necessarily the
+# one whose own request produced that wakeup. A thread woken by someone else's signal
+# finds nothing left in the (already-drained) queue, so it skips its own inner timeout
+# check and falls into the buggy tail check, raising `TypeError: nil can't be coerced
+# into Float` instead of the intended timeout/empty result. Reproduced deterministically
+# with N threads fetching in a loop against the same PullSubscription.
+#
+# Second, separate upstream bug in the same method: the post-wait timeout check below
+# (`raise ... if MonotonicTime.since(t) > timeout`) fires unconditionally, unlike every
+# other timeout check in this method (all guarded with `msgs.empty? &&`). If the unsynchronized
+# pop just above it succeeded in grabbing a real, already-delivered message -- which can happen
+# even after `timeout` has technically elapsed, since #dispatch's signal and this thread actually
+# resuming are two different moments -- that message is already gone from @pending_queue with
+# nowhere else to go, yet gets discarded here and a plain NATS::Timeout raised instead. Callers
+# (including Cosmo::Processor#fetch) treat NATS::Timeout as "no messages", so the message is lost
+# silently: never processed, never acked, and not redelivered until the consumer's ack_wait expires.
+# Reproduces reliably with several threads fetching (batch: 1) concurrently against the same
+# subscription, e.g. Cosmo::Job::Processor's thread pool.
+#
+# Vendored copy of nats-pure's PullSubscription#fetch, kept as close to
+# upstream as possible (two one-line fixes, see comments above), so it's easy to diff against the
+# next nats-pure release and drop once fixed there. Fixed on nats-pure main by the fetch rewrite
+# (#181), recognized by its private #take_pending, in which case this copy is not loaded.
+return if NATS::JetStream.const_get(:PullSubscription).private_method_defined?(:take_pending)
+
+# rubocop:disable all
+module NATS
+  class JetStream
+    module PullSubscription
+      def fetch(batch = 1, params = {})
+        raise ::NATS::JetStream::Error.new("nats: invalid batch size") if batch < 1
+
+        t = MonotonicTime.now
+        start_time = t # fixes the upstream bug -- see comment above
+        timeout = params[:timeout] ||= 5
+        expires = (timeout * 1_000_000_000) - 100_000
+        next_req = { batch: batch }
+
+        msgs = []
+        case
+        when batch == 1
+          synchronize do
+            unless @pending_queue.empty?
+              msg = @pending_queue.pop
+              @pending_size -= msg.data.size
+              if JS.is_status_msg(msg)
+                case msg.header["Status"]
+                when JS::Status::NoMsgs then nil
+                when JS::Status::RequestTimeout then nil # Skip
+                else raise JS.from_msg(msg)
+                end
+              else
+                msgs << msg
+              end
+            end
+          end
+
+          next_req[:expires] = expires
+          if msgs.empty?
+            @nc.publish(@jsi.nms, JS.next_req_to_json(next_req), @subject)
+            synchronize { wait_for_msgs_cond.wait(timeout) }
+
+            unless @pending_queue.empty?
+              msg = @pending_queue.pop
+              @pending_size -= msg.data.size
+              msgs << msg
+            end
+
+            raise ::NATS::Timeout.new("nats: fetch timeout") if msgs.empty? && (MonotonicTime.since(t) > timeout)
+
+            if JS.is_status_msg(msgs.first)
+              msg = msgs.first
+              case msg.header[JS::Header::Status]
+              when JS::Status::RequestTimeout then raise NATS::Timeout.new("nats: fetch request timeout")
+              else raise JS.from_msg(msgs.first)
+              end
+            end
+          end
+        when batch > 1
+          synchronize do
+            if batch <= @pending_queue.size
+              batch.times do
+                msg = @pending_queue.pop
+                @pending_size -= msg.data.size
+                if JS.is_status_msg(msg)
+                  case msg.header[JS::Header::Status]
+                  when JS::Status::NoMsgs, JS::Status::RequestTimeout then next
+                  else raise JS.from_msg(msg)
+                  end
+                else
+                  msgs << msg
+                end
+              end
+              return msgs
+            end
+          end
+
+          next_req[:no_wait] = true
+          @nc.publish(@jsi.nms, JS.next_req_to_json(next_req), @subject)
+
+          start_time = MonotonicTime.now
+          msg = nil
+
+          synchronize do
+            wait_for_msgs_cond.wait(timeout)
+            unless @pending_queue.empty?
+              msg = @pending_queue.pop
+              @pending_size -= msg.data.size
+            end
+          end
+
+          if !msg.nil? && JS.is_status_msg(msg)
+            case msg.header[JS::Header::Status]
+            when JS::Status::NoMsgs
+              next_req[:expires] = expires
+              next_req.delete(:no_wait)
+              @nc.publish(@jsi.nms, JS.next_req_to_json(next_req), @subject)
+            when JS::Status::RequestTimeout
+              raise NATS::Timeout.new("nats: fetch request timeout")
+            else
+              raise JS.from_msg(msg)
+            end
+          else
+            msgs << msg unless msg.nil?
+          end
+
+          duration = MonotonicTime.since(start_time)
+          raise NATS::Timeout.new("nats: fetch timeout") if msgs.empty? && (duration > timeout)
+
+          needed = batch - msgs.count
+          while (needed > 0) && (MonotonicTime.since(start_time) < timeout)
+            duration = MonotonicTime.since(start_time)
+
+            synchronize do
+              if @pending_queue.empty?
+                deadline = timeout - duration
+                wait_for_msgs_cond.wait(deadline) if deadline > 0
+
+                duration = MonotonicTime.since(start_time)
+                if msgs.empty? && @pending_queue.empty? && (duration > timeout)
+                  raise NATS::Timeout.new("nats: fetch timeout")
+                end
+              end
+
+              unless @pending_queue.empty?
+                msg = @pending_queue.pop
+                @pending_size -= msg.data.size
+
+                if JS.is_status_msg(msg)
+                  case msg.header[JS::Header::Status]
+                  when JS::Status::NoMsgs, JS::Status::RequestTimeout
+                    duration = MonotonicTime.since(start_time)
+                    if duration > timeout
+                      raise NATS::Timeout.new("nats: fetch timeout") if msgs.empty?
+
+                      return msgs
+                    end
+                  else
+                    raise JS.from_msg(msg)
+                  end
+                else
+                  msgs << msg
+                  needed -= 1
+                end
+              end
+            end
+          end
+        end
+
+        raise ::NATS::Timeout.new("nats: fetch timeout") if msgs.empty? && (MonotonicTime.since(start_time) > timeout)
+
+        msgs
+      end
+    end
+  end
 end
+# rubocop:enable all

data/lib/cosmo/utils/thread_pool.rb

@@ -16,11 +16,16 @@ module Cosmo
 
       def initialize(concurrency)
         @mutex = Thread::Mutex.new
+        @concurrency = concurrency
         @available = concurrency
         @cond = ConditionVariable.new
         @pool = Concurrent::FixedThreadPool.new(concurrency)
       end
 
+      def wait_idle
+        @mutex.synchronize { @cond.wait(@mutex) until @available == @concurrency }
+      end
+
       def post
         @mutex.synchronize do
           @cond.wait(@mutex) while @available <= 0

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.5.1"
+  VERSION = "0.7.0"
 end

data/lib/cosmo/web/assets/app.css

@@ -163,6 +163,14 @@ button:hover, .btn:hover {
   );
 }
 
+.btn-sm {
+  box-shadow: none;
+  font-size: var(--font-size-small);
+  font-weight: 600;
+  line-height: var(--space-2x);
+  padding: var(--space-1-2) var(--space);
+}
+
 /* ── Badges ────────────────────────────────────────────────────────────── */
 .badge {
   border-radius: var(--space-1-2);
@@ -343,6 +351,12 @@ section .pending {
   align-items: center;
 }
 
+section .pending .poll-status {
+  align-items: center;
+  display: inline-flex;
+  gap: var(--space);
+}
+
 /* ── Stat cards ────────────────────────────────────────────────────────── */
 .cards-container {
   display: grid;
@@ -479,6 +493,13 @@ time {
   font-size: var(--font-size-small);
   word-wrap: break-word;
 }
+.error-message details code {
+  max-width: 600px;
+  max-height: 320px;
+  overflow: auto;
+  white-space: pre;
+  font-size: var(--font-size-small);
+}
 
 /* ── Links ─────────────────────────────────────────────────────────────── */
 .stream-link { color: var(--color-primary); font-weight: 600; }

data/lib/cosmo/web/controllers/jobs.rb

@@ -54,6 +54,13 @@ module Cosmo
           ok
         end
 
+        def delete_enqueued
+          seq = path.split("/").last.to_i
+          stream_name, = streams
+          API::Stream.new(stream_name).delete(seq)
+          _enqueued
+        end
+
         def _scheduled
           stream = API::Stream.new("scheduled")
           jobs = stream.messages(page: params["page"], limit: params["limit"])
@@ -67,9 +74,19 @@ module Cosmo
         end
 
         def _busy
-          limit = (limit || 25).to_i
-          jobs  = API::Busy.instance.list(limit:)
-          ok render("jobs/_busy", { jobs: jobs, total: API::Busy.instance.size })
+          busy = API::Stats::Busy.instance
+          total = busy.size
+          page, limit, total_pages = paginate(total, API::Stats::Busy::LIMIT)
+          polling = params["poll"].to_s != "0"
+          ok render("jobs/_busy", { jobs: busy.list(page:, limit:), total:, page:, limit:, total_pages:, polling: })
+        end
+
+        def _processes
+          processes = API::Stats::Processes.instance
+          total = processes.size
+          page, limit, total_pages = paginate(total, API::Stats::Processes::LIMIT)
+          polling = params["poll"].to_s != "0"
+          ok render("jobs/_processes", { processes: processes.list(page:, limit:), total:, page:, limit:, total_pages:, polling: })
         end
 
         def _enqueued # rubocop:disable Metrics/AbcSize
@@ -80,8 +97,9 @@ module Cosmo
           unless stream_name.to_s.empty?
             stream = API::Stream.new(stream_name)
             total = stream.total
-            jobs = stream.messages(page:, limit:)
             total_pages = (total.to_f / limit).ceil
+            page = page.clamp(1, [total_pages, 1].max)
+            jobs = stream.messages(page:, limit:)
           end
 
           ok render("jobs/_enqueued", { jobs:, total:, stream_name:, stream_names:, page:, limit:, total_pages: })
@@ -93,6 +111,12 @@ module Cosmo
 
         private
 
+        def paginate(total, default_limit)
+          limit = (params["limit"] || default_limit).to_i
+          total_pages = (total.to_f / limit).ceil
+          [params["page"].to_i.clamp(1, [total_pages, 1].max), limit, total_pages]
+        end
+
         def streams
           stream_names = API::Stream.jobs.map(&:name)
           stream_name = params.fetch("stream_name", stream_names.first)

data/lib/cosmo/web/helpers/application.rb

@@ -8,6 +8,13 @@ module Cosmo
       module Application
         include Renderer
 
+        # Seconds between htmx auto-refreshes. Override with COSMO_WEB_POLL_INTERVAL.
+        POLL_INTERVAL = 5
+
+        def poll_interval
+          @poll_interval ||= ENV.fetch("COSMO_WEB_POLL_INTERVAL", POLL_INTERVAL).to_i
+        end
+
         def render(template, locals = nil)
           defaults = { request: @request }
           locals = Hash(locals).merge(defaults)
@@ -42,8 +49,10 @@ module Cosmo
             "#{elapsed}s"
           elsif elapsed < 3600
             "#{elapsed / 60}m #{elapsed % 60}s"
-          else
+          elsif elapsed < 86_400
             "#{elapsed / 3600}h #{(elapsed % 3600) / 60}m"
+          else
+            "#{elapsed / 86_400}d #{(elapsed % 86_400) / 3600}h"
           end
         end
 

data/lib/cosmo/web/views/batches/index.erb

@@ -3,7 +3,7 @@
 
   <div id="batches-table"
        hx-get="<%= url_for('/batches/_table') %>"
-       hx-trigger="load, every 5s"
+       hx-trigger="load, every <%= poll_interval %>s"
        hx-swap="innerHTML">
     <div class="alert alert-info">Loading batches&hellip;</div>
   </div>

data/lib/cosmo/web/views/jobs/_busy.erb

@@ -1,10 +1,29 @@
+<% nav = @polling ? { limit: @limit } : { limit: @limit, poll: 0 } -%>
 <div id="busy-poller"
-     hx-get="<%= url_for('/jobs/_busy') %>"
-     hx-trigger="every 5s"
-     hx-swap="outerHTML">
+     <% if @polling -%>
+     hx-get="<%= url_for('/jobs/_busy', page: @page, limit: @limit) %>"
+     hx-trigger="every <%= poll_interval %>s"
+     hx-swap="outerHTML"
+     <% end -%>
+     >
   <div class="pending">
     <span></span>
-    <span class="text-muted"><%= @total %> job(s) running</span>
+    <span class="poll-status">
+      <span class="text-muted"><%= @total %> job(s) running</span>
+      <% if @polling -%>
+        <a class="btn btn-sm"
+           hx-get="<%= url_for('/jobs/_busy', nav.merge(page: @page, poll: 0)) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner">&#9209; Stop Polling</a>
+      <% else -%>
+        <a class="btn btn-sm btn-primary"
+           hx-get="<%= url_for('/jobs/_busy', page: @page, limit: @limit) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner">&#x25B6; Live Poll</a>
+      <% end -%>
+    </span>
   </div>
 
   <% if @jobs.empty? -%>
@@ -18,6 +37,7 @@
             <th>JID[Args]</th>
             <th>Stream</th>
             <th>Worker</th>
+            <th>Attempt</th>
             <th>Running for</th>
           </tr>
         </thead>
@@ -43,6 +63,14 @@
               <code><%= h(job[:stream].to_s) %></code>
             </td>
             <td><code style="font-size: var(--font-size-small);"><%= h(job[:worker].to_s) %></code></td>
+            <td>
+              <% delivery = job[:delivery].to_i -%>
+              <% if delivery > 1 -%>
+                <span class="badge <%= delivery >= 5 ? 'badge-danger' : 'badge-warning' %>"><%= delivery %></span>
+              <% else -%>
+                <span class="text-muted"><%= delivery.zero? ? '&mdash;' : delivery %></span>
+              <% end -%>
+            </td>
             <td>
               <%= elapsed(job[:started_at]) %>
             </td>
@@ -52,4 +80,42 @@
       </table>
     </div>
   <% end -%>
+
+  <% if @total_pages.to_i > 1 -%>
+    <div class="pagination">
+      <% if @page > 1 -%>
+        <a hx-get="<%= url_for('/jobs/_busy', nav.merge(page: @page - 1)) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner"
+           class="btn">&#x2190; Prev</a>
+      <% else -%>
+        <span class="btn btn-disabled">&#x2190; Prev</span>
+      <% end -%>
+
+      <% pages(@page, @total_pages).each do |p| -%>
+        <% if p == :gap -%>
+          <span class="pagination-gap">&hellip;</span>
+        <% elsif p == @page -%>
+          <span class="btn btn-primary btn-disabled"><%= p %></span>
+        <% else -%>
+          <a hx-get="<%= url_for('/jobs/_busy', nav.merge(page: p)) %>"
+             hx-target="#busy-poller"
+             hx-swap="outerHTML"
+             hx-indicator="#global-spinner"
+             class="btn"><%= p %></a>
+        <% end -%>
+      <% end -%>
+
+      <% if @page < @total_pages -%>
+        <a hx-get="<%= url_for('/jobs/_busy', nav.merge(page: @page + 1)) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner"
+           class="btn">Next &#x2192;</a>
+      <% else -%>
+        <span class="btn btn-disabled">Next &#x2192;</span>
+      <% end -%>
+    </div>
+  <% end -%>
 </div>

data/lib/cosmo/web/views/jobs/_dead.erb

@@ -1,7 +1,4 @@
-<div id="dead-poller"
-     hx-get="<%= url_for('/jobs/_dead') %>"
-     hx-trigger="every 5s"
-     hx-swap="outerHTML">
+<div id="dead-jobs">
   <% if @jobs.empty? -%>
     <div class="alert alert-success">No dead jobs found</div>
   <% else -%>
@@ -35,31 +32,47 @@
                 </details>
               </div>
             </td>
-            <td>
-              <code><%= h(job.stream) %></code>
+            <td style="white-space: nowrap;">
+              <% if job.x_stream -%><code><%= h(job.x_stream) %></code><% else -%><span class="text-muted">&mdash;</span><% end -%>
             </td>
-            <td style="max-width: 400px;">
-              <% if job.data[:error] -%>
+            <td style="min-width: 420px;">
+              <% if job.error? -%>
               <div class="error-message">
-                <strong><%= h(job.data[:error].to_s.split(":").first || "Error") %></strong>
-                <p><%= h(job.data[:error]) %></p>
+                <strong><%= h(job.error_class.to_s.empty? ? "Error" : job.error_class) %></strong>
+                <p><%= h(job.error_message) %></p>
+                <% if job.error_backtrace -%>
+                <details>
+                  <summary>Backtrace</summary>
+                  <code><%= h(job.error_backtrace_lines.join("\n")) %></code>
+                </details>
+                <% end -%>
+              </div>
+              <% else -%>
+              <div class="error-message">
+                <strong>Unknown</strong>
+                <p>No error recorded for this job</p>
               </div>
               <% end -%>
             </td>
             <td style="white-space: nowrap;">
-              <time><%= h(job.timestamp) %></time>
+              <% if (failed_at = job.stored_at) -%>
+              <time datetime="<%= h(job.timestamp) %>" title="<%= h(job.timestamp) %>"><%= h(format_timestamp(failed_at)) %></time>
+              <div class="text-muted" style="font-size: var(--font-size-small);"><%= elapsed(failed_at) %> ago</div>
+              <% else -%>
+              <span class="text-muted">&mdash;</span>
+              <% end -%>
             </td>
-            <td>
+            <td style="white-space: nowrap;">
               <a hx-patch="<%= url_for("/jobs/retry/#{job.seq}") %>"
                  hx-target="#dead-row-<%= job.seq %>"
                  hx-swap="innerHTML"
-                 class="btn btn-primary"
-                 style="width: 100%; text-align: center; margin-bottom: 3px">&#x21BB; Retry</a>
+                 title="Retry"
+                 class="btn btn-primary btn-sm">&#x21BB;</a>
               <a hx-delete="<%= url_for("/jobs/delete/#{job.seq}") %>"
                  hx-target="#dead-row-<%= job.seq %>"
                  hx-swap="innerHTML"
-                 class="btn btn-danger"
-                 style="width: 100%; text-align: center; margin-bottom: 3px">&#x2715; Delete</a>
+                 title="Delete"
+                 class="btn btn-danger btn-sm">&#x2715;</a>
             </td>
           </tr>
           <% end -%>

data/lib/cosmo/web/views/jobs/_enqueued.erb

@@ -30,6 +30,7 @@
             <th>JID[Args]</th>
             <th>Stream</th>
             <th>Seq</th>
+            <th>Actions</th>
           </tr>
         </thead>
         <tbody>
@@ -56,6 +57,15 @@
             <td>
               <code><%= job.seq %></code>
             </td>
+            <td>
+              <a hx-delete="<%= url_for("/jobs/enqueued/#{job.seq}", { stream_name: @stream_name, page: @page, limit: @limit }) %>"
+                 hx-target="#enqueued-poller"
+                 hx-swap="outerHTML"
+                 hx-indicator="#global-spinner"
+                 hx-confirm="Remove message #<%= job.seq %> from <%= h(@stream_name) %>?"
+                 title="Remove message"
+                 class="btn btn-danger btn-sm">&#x2715;</a>
+            </td>
           </tr>
           <% end -%>
         </tbody>