cosmonats 0.6.0 → 0.8.0

data/lib/cosmo/web/chart.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class Web
+    # Geometry for the Metrics tab's SVG chart: a bar per day for the number of runs on the left axis, and a line for
+    # the average execution time on the right one.
+    class Chart
+      WIDTH = 760
+      HEIGHT = 240
+      TOP = 16
+      BOTTOM = 212
+      LEFT = 52
+      RIGHT = 700
+      TICKS = 4
+      LABELS = 10
+
+      # @param series [Array<Hash>] +{ date:, count:, exec_ms: }+ per day, oldest first
+      def initialize(series)
+        @series = series
+        @count_top = top(series.map { _1[:count] }.max.to_i)
+        @exec_top = top(series.filter_map { _1[:exec_ms] }.max.to_f)
+      end
+
+      # @return [Boolean]
+      def empty?
+        @series.all? { _1[:count].zero? }
+      end
+
+      # @return [Array<Hash>] +{ x:, y:, width:, height:, title: }+
+      def bars # rubocop:disable Metrics/AbcSize
+        @series.each_with_index.map do |day, index|
+          height = scale(day[:count], @count_top)
+          { x: (slot_x(index) + (slot * 0.15)).round(1), y: (BOTTOM - height).round(1), width: (slot * 0.7).round(1),
+            height: height.round(1), title: "#{label(day[:date])}: #{day[:count]} run(s)" }
+        end
+      end
+
+      # @return [Array<Hash>] +{ x:, y:, title: }+ for the days with an average
+      def points
+        @series.each_with_index.filter_map do |day, index|
+          next unless day[:exec_ms]
+
+          { x: (slot_x(index) + (slot / 2)).round(1), y: (BOTTOM - scale(day[:exec_ms], @exec_top)).round(1),
+            title: "#{label(day[:date])}: #{day[:exec_ms]} ms on average" }
+        end
+      end
+
+      # @return [String] the +points+ attribute of the execution time polyline
+      def line
+        points.map { "#{_1[:x]},#{_1[:y]}" }.join(" ")
+      end
+
+      # @return [Array<Hash>] +{ y:, count:, exec_ms: }+ per horizontal grid line, bottom up
+      def ticks
+        (0..TICKS).map do |step|
+          { y: (BOTTOM - ((BOTTOM - TOP) * step / TICKS.to_f)).round(1),
+            count: (@count_top * step / TICKS.to_f).round, exec_ms: (@exec_top * step / TICKS.to_f).round }
+        end
+      end
+
+      # @return [Array<Hash>] +{ x:, text: }+, thinned out to at most LABELS
+      def labels
+        every = (@series.size / LABELS.to_f).ceil.clamp(1, nil)
+        @series.each_with_index.filter_map do |day, index|
+          { x: (slot_x(index) + (slot / 2)).round(1), text: label(day[:date]) } if (index % every).zero?
+        end
+      end
+
+      private
+
+      def slot
+        (RIGHT - LEFT) / [@series.size, 1].max.to_f
+      end
+
+      def slot_x(index)
+        LEFT + (slot * index)
+      end
+
+      def scale(value, top)
+        top.zero? ? 0 : (BOTTOM - TOP) * value / top.to_f
+      end
+
+      # The smallest "nice" axis maximum (1, 2 or 5 times a power of ten per tick) at or above +max+.
+      def top(max)
+        return TICKS if max.zero?
+
+        raw = max / TICKS.to_f
+        magnitude = 10**Math.log10(raw).floor
+        step = [1, 2, 5, 10].map { _1 * magnitude }.find { _1 >= raw }
+        step * TICKS
+      end
+
+      def label(date)
+        "#{date[4, 2]}/#{date[6, 2]}"
+      end
+    end
+  end
+end

data/lib/cosmo/web/controllers/crons.rb

@@ -7,17 +7,23 @@ module Cosmo
     module Controllers
       class Crons < Application
         def index
+          return not_found unless Config.scheduled.enabled
+
           content_for :title, "Crons"
           ok render("crons/index", layout: true)
         end
 
         def _table
+          return not_found unless Config.scheduled.enabled
+
           ok render("crons/_table", { schedules: cron.all })
         end
 
         # Dispatch the job immediately, bypassing the schedule timer.
         # Expects params["subject"] = the schedule subject stored in NATS.
         def run_now
+          return not_found unless Config.scheduled.enabled
+
           subject = Rack::Utils.unescape(params["subject"].to_s)
           cron.run_now!(subject)
           ok
@@ -25,6 +31,8 @@ module Cosmo
 
         # Purge the schedule from NATS so it stops firing.
         def delete
+          return not_found unless Config.scheduled.enabled
+
           subject = Rack::Utils.unescape(params["subject"].to_s)
           cron.delete!(subject)
           _table

data/lib/cosmo/web/controllers/jobs.rb

@@ -8,7 +8,8 @@ module Cosmo
       class Jobs < Application
         def index
           content_for :title, "Jobs"
-          ok render("jobs/index", layout: true)
+          processes_query = params.slice("page", "limit").transform_values(&:to_i).select { _2.positive? }
+          ok render("jobs/index", { processes_query: }, layout: true)
         end
 
         def busy
@@ -27,6 +28,8 @@ module Cosmo
         end
 
         def scheduled
+          return not_found unless Config.scheduled.enabled
+
           return _scheduled if hx_request?
 
           content_for :title, "Scheduled Jobs"
@@ -34,6 +37,8 @@ module Cosmo
         end
 
         def dead
+          return not_found unless Config.dead.enabled
+
           return _dead if hx_request?
 
           content_for :title, "Dead Jobs"
@@ -41,6 +46,8 @@ module Cosmo
         end
 
         def retry
+          return not_found unless Config.dead.enabled
+
           seq = path.split("/").last.to_i
           stream = API::Stream.new("dead")
           stream.retry(seq)
@@ -48,28 +55,52 @@ module Cosmo
         end
 
         def delete
+          return not_found unless Config.dead.enabled
+
           seq = path.split("/").last.to_i
           stream = API::Stream.new("dead")
           stream.delete(seq)
           ok
         end
 
+        def delete_enqueued
+          seq = path.split("/").last.to_i
+          stream_name, = streams
+          API::Stream.new(stream_name).delete(seq)
+          _enqueued
+        end
+
         def _scheduled
+          return not_found unless Config.scheduled.enabled
+
           stream = API::Stream.new("scheduled")
           jobs = stream.messages(page: params["page"], limit: params["limit"])
           ok render("jobs/_scheduled", { jobs: jobs, total: stream.total })
         end
 
         def _dead
+          return not_found unless Config.dead.enabled
+
           stream = API::Stream.new("dead")
           jobs = stream.messages(page: params["page"], limit: params["limit"])
           ok render("jobs/_dead", { jobs: jobs, total: stream.total })
         end
 
         def _busy
-          limit = (limit || 25).to_i
-          jobs  = API::Busy.instance.list(limit:)
-          ok render("jobs/_busy", { jobs: jobs, total: API::Busy.instance.size })
+          busy = API::Stats::Busy.instance
+          total = busy.size
+          page, limit, total_pages = paginate(total, API::Stats::Busy::LIMIT)
+          polling = params["poll"].to_s != "0"
+          ok render("jobs/_busy", { jobs: busy.list(page:, limit:), total:, page:, limit:, total_pages:, polling: })
+        end
+
+        def _processes
+          processes = API::Stats::Processes.instance
+          total = processes.size
+          page, limit, total_pages = paginate(total, API::Stats::Processes::LIMIT)
+          polling = params["poll"].to_s != "0"
+          groups = processes.list(page:, limit:).chunk { API::Stats::Processes.kind(_1) }.to_a
+          ok render("jobs/_processes", { groups:, total:, page:, limit:, total_pages:, polling: })
         end
 
         def _enqueued # rubocop:disable Metrics/AbcSize
@@ -80,8 +111,9 @@ module Cosmo
           unless stream_name.to_s.empty?
             stream = API::Stream.new(stream_name)
             total = stream.total
-            jobs = stream.messages(page:, limit:)
             total_pages = (total.to_f / limit).ceil
+            page = page.clamp(1, [total_pages, 1].max)
+            jobs = stream.messages(page:, limit:)
           end
 
           ok render("jobs/_enqueued", { jobs:, total:, stream_name:, stream_names:, page:, limit:, total_pages: })
@@ -93,6 +125,12 @@ module Cosmo
 
         private
 
+        def paginate(total, default_limit)
+          limit = (params["limit"] || default_limit).to_i
+          total_pages = (total.to_f / limit).ceil
+          [params["page"].to_i.clamp(1, [total_pages, 1].max), limit, total_pages]
+        end
+
         def streams
           stream_names = API::Stream.jobs.map(&:name)
           stream_name = params.fetch("stream_name", stream_names.first)

data/lib/cosmo/web/controllers/metrics.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "cosmo/web/controllers/application"
+require "cosmo/web/chart"
+
+module Cosmo
+  class Web
+    module Controllers
+      class Metrics < Application
+        RANGES = { 1 => "Today", 7 => "7 days", 30 => "30 days" }.freeze
+
+        def index
+          return not_found unless Config.metrics.enabled
+
+          content_for :title, "Metrics"
+          ok render("metrics/index", { days:, ranges: RANGES }, layout: true)
+        end
+
+        # The chart and the table together, so the charted job stays highlighted across refreshes.
+        def _panel
+          return not_found unless Config.metrics.enabled
+
+          rows = metrics.summary(days:)
+          job = charted(rows)
+          chart = Chart.new(metrics.daily(job, days:)) if job && days > 1
+          ok render("metrics/_panel", { rows:, job:, days:, chart: })
+        end
+
+        private
+
+        # The requested job while it has rows in the period, the busiest one otherwise.
+        def charted(rows)
+          job = params["job"].to_s
+          rows.any? { _1[:job] == job } ? job : rows.first&.dig(:job)
+        end
+
+        def days
+          value = params["days"].to_i
+          RANGES.key?(value) ? value : 7
+        end
+
+        def metrics
+          API::Stats::Metrics.instance
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/web/helpers/application.rb

@@ -8,6 +8,13 @@ module Cosmo
       module Application
         include Renderer
 
+        # Seconds between htmx auto-refreshes. Override with COSMO_WEB_POLL_INTERVAL.
+        POLL_INTERVAL = 5
+
+        def poll_interval
+          @poll_interval ||= ENV.fetch("COSMO_WEB_POLL_INTERVAL", POLL_INTERVAL).to_i
+        end
+
         def render(template, locals = nil)
           defaults = { request: @request }
           locals = Hash(locals).merge(defaults)
@@ -35,6 +42,16 @@ module Cosmo
           value.to_s
         end
 
+        # @param millis [Numeric, nil]
+        # @return [String] e.g. "340 ms", "1.25 s", "2m 3s", or "—" without a value
+        def duration(millis)
+          return "—" unless millis
+          return "#{millis.round} ms" if millis < 1_000
+          return "#{(millis / 1_000.0).round(2)} s" if millis < 60_000
+
+          "#{(millis / 60_000).floor}m #{((millis % 60_000) / 1_000).round}s"
+        end
+
         def elapsed(value)
           elapsed = Time.now.to_i - value.to_i
 
@@ -42,8 +59,10 @@ module Cosmo
             "#{elapsed}s"
           elsif elapsed < 3600
             "#{elapsed / 60}m #{elapsed % 60}s"
-          else
+          elsif elapsed < 86_400
             "#{elapsed / 3600}h #{(elapsed % 3600) / 60}m"
+          else
+            "#{elapsed / 86_400}d #{(elapsed % 86_400) / 3600}h"
           end
         end
 

data/lib/cosmo/web/views/batches/_table.erb

@@ -8,7 +8,9 @@
           <th>Batch</th>
           <th>Parent</th>
           <th>Status</th>
+          <th>Progress</th>
           <th>Total</th>
+          <th>Pending</th>
           <th>Succeeded</th>
           <th>Failed</th>
           <th>Created</th>
@@ -17,6 +19,7 @@
       <tbody>
         <% @batches.each do |b| -%>
           <% stats = b.stats -%>
+          <% total = stats[:total].to_i -%>
           <tr>
             <td><code><%= h(b.bid) %></code></td>
             <td>
@@ -35,7 +38,15 @@
                 <span class="badge badge-success">Success</span>
               <% end -%>
             </td>
+            <td>
+              <% if total.positive? -%>
+                <%= ((total - stats[:pending].to_i) * 100 / total).clamp(0, 100) %>%
+              <% else -%>
+                <span class="text-muted">&mdash;</span>
+              <% end -%>
+            </td>
             <td><%= format_numbers(stats[:total]) %></td>
+            <td><%= format_numbers(stats[:pending]) %></td>
             <td><%= format_numbers(stats[:succeeded]) %></td>
             <td><%= format_numbers(stats[:failed]) %></td>
             <td><%= format_timestamp(b.created_at) %></td>

data/lib/cosmo/web/views/batches/index.erb

@@ -3,7 +3,7 @@
 
   <div id="batches-table"
        hx-get="<%= url_for('/batches/_table') %>"
-       hx-trigger="load, every 5s"
+       hx-trigger="load, every <%= poll_interval %>s"
        hx-swap="innerHTML">
     <div class="alert alert-info">Loading batches&hellip;</div>
   </div>

data/lib/cosmo/web/views/jobs/_busy.erb

@@ -1,10 +1,29 @@
+<% nav = @polling ? { limit: @limit } : { limit: @limit, poll: 0 } -%>
 <div id="busy-poller"
-     hx-get="<%= url_for('/jobs/_busy') %>"
-     hx-trigger="every 5s"
-     hx-swap="outerHTML">
+     <% if @polling -%>
+     hx-get="<%= url_for('/jobs/_busy', page: @page, limit: @limit) %>"
+     hx-trigger="every <%= poll_interval %>s"
+     hx-swap="outerHTML"
+     <% end -%>
+     >
   <div class="pending">
     <span></span>
-    <span class="text-muted"><%= @total %> job(s) running</span>
+    <span class="poll-status">
+      <span class="text-muted"><%= @total %> job(s) running</span>
+      <% if @polling -%>
+        <a class="btn btn-sm"
+           hx-get="<%= url_for('/jobs/_busy', nav.merge(page: @page, poll: 0)) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner">&#9209; Stop Polling</a>
+      <% else -%>
+        <a class="btn btn-sm btn-primary"
+           hx-get="<%= url_for('/jobs/_busy', page: @page, limit: @limit) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner">&#x25B6; Live Poll</a>
+      <% end -%>
+    </span>
   </div>
 
   <% if @jobs.empty? -%>
@@ -18,6 +37,7 @@
             <th>JID[Args]</th>
             <th>Stream</th>
             <th>Worker</th>
+            <th>Attempt</th>
             <th>Running for</th>
           </tr>
         </thead>
@@ -43,6 +63,14 @@
               <code><%= h(job[:stream].to_s) %></code>
             </td>
             <td><code style="font-size: var(--font-size-small);"><%= h(job[:worker].to_s) %></code></td>
+            <td>
+              <% delivery = job[:delivery].to_i -%>
+              <% if delivery > 1 -%>
+                <span class="badge <%= delivery >= 5 ? 'badge-danger' : 'badge-warning' %>"><%= delivery %></span>
+              <% else -%>
+                <span class="text-muted"><%= delivery.zero? ? '&mdash;' : delivery %></span>
+              <% end -%>
+            </td>
             <td>
               <%= elapsed(job[:started_at]) %>
             </td>
@@ -52,4 +80,42 @@
       </table>
     </div>
   <% end -%>
+
+  <% if @total_pages.to_i > 1 -%>
+    <div class="pagination">
+      <% if @page > 1 -%>
+        <a hx-get="<%= url_for('/jobs/_busy', nav.merge(page: @page - 1)) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner"
+           class="btn">&#x2190; Prev</a>
+      <% else -%>
+        <span class="btn btn-disabled">&#x2190; Prev</span>
+      <% end -%>
+
+      <% pages(@page, @total_pages).each do |p| -%>
+        <% if p == :gap -%>
+          <span class="pagination-gap">&hellip;</span>
+        <% elsif p == @page -%>
+          <span class="btn btn-primary btn-disabled"><%= p %></span>
+        <% else -%>
+          <a hx-get="<%= url_for('/jobs/_busy', nav.merge(page: p)) %>"
+             hx-target="#busy-poller"
+             hx-swap="outerHTML"
+             hx-indicator="#global-spinner"
+             class="btn"><%= p %></a>
+        <% end -%>
+      <% end -%>
+
+      <% if @page < @total_pages -%>
+        <a hx-get="<%= url_for('/jobs/_busy', nav.merge(page: @page + 1)) %>"
+           hx-target="#busy-poller"
+           hx-swap="outerHTML"
+           hx-indicator="#global-spinner"
+           class="btn">Next &#x2192;</a>
+      <% else -%>
+        <span class="btn btn-disabled">Next &#x2192;</span>
+      <% end -%>
+    </div>
+  <% end -%>
 </div>

data/lib/cosmo/web/views/jobs/_dead.erb

@@ -1,7 +1,4 @@
-<div id="dead-poller"
-     hx-get="<%= url_for('/jobs/_dead') %>"
-     hx-trigger="every 5s"
-     hx-swap="outerHTML">
+<div id="dead-jobs">
   <% if @jobs.empty? -%>
     <div class="alert alert-success">No dead jobs found</div>
   <% else -%>
@@ -35,31 +32,47 @@
                 </details>
               </div>
             </td>
-            <td>
-              <code><%= h(job.stream) %></code>
+            <td style="white-space: nowrap;">
+              <% if job.x_stream -%><code><%= h(job.x_stream) %></code><% else -%><span class="text-muted">&mdash;</span><% end -%>
             </td>
-            <td style="max-width: 400px;">
-              <% if job.data[:error] -%>
+            <td style="min-width: 420px;">
+              <% if job.error? -%>
               <div class="error-message">
-                <strong><%= h(job.data[:error].to_s.split(":").first || "Error") %></strong>
-                <p><%= h(job.data[:error]) %></p>
+                <strong><%= h(job.error_class.to_s.empty? ? "Error" : job.error_class) %></strong>
+                <p><%= h(job.error_message) %></p>
+                <% if job.error_backtrace -%>
+                <details>
+                  <summary>Backtrace</summary>
+                  <code><%= h(job.error_backtrace_lines.join("\n")) %></code>
+                </details>
+                <% end -%>
+              </div>
+              <% else -%>
+              <div class="error-message">
+                <strong>Unknown</strong>
+                <p>No error recorded for this job</p>
               </div>
               <% end -%>
             </td>
             <td style="white-space: nowrap;">
-              <time><%= h(job.timestamp) %></time>
+              <% if (failed_at = job.stored_at) -%>
+              <time datetime="<%= h(job.timestamp) %>" title="<%= h(job.timestamp) %>"><%= h(format_timestamp(failed_at)) %></time>
+              <div class="text-muted" style="font-size: var(--font-size-small);"><%= elapsed(failed_at) %> ago</div>
+              <% else -%>
+              <span class="text-muted">&mdash;</span>
+              <% end -%>
             </td>
-            <td>
+            <td style="white-space: nowrap;">
               <a hx-patch="<%= url_for("/jobs/retry/#{job.seq}") %>"
                  hx-target="#dead-row-<%= job.seq %>"
                  hx-swap="innerHTML"
-                 class="btn btn-primary"
-                 style="width: 100%; text-align: center; margin-bottom: 3px">&#x21BB; Retry</a>
+                 title="Retry"
+                 class="btn btn-primary btn-sm">&#x21BB;</a>
               <a hx-delete="<%= url_for("/jobs/delete/#{job.seq}") %>"
                  hx-target="#dead-row-<%= job.seq %>"
                  hx-swap="innerHTML"
-                 class="btn btn-danger"
-                 style="width: 100%; text-align: center; margin-bottom: 3px">&#x2715; Delete</a>
+                 title="Delete"
+                 class="btn btn-danger btn-sm">&#x2715;</a>
             </td>
           </tr>
           <% end -%>

data/lib/cosmo/web/views/jobs/_enqueued.erb

@@ -30,6 +30,7 @@
             <th>JID[Args]</th>
             <th>Stream</th>
             <th>Seq</th>
+            <th>Actions</th>
           </tr>
         </thead>
         <tbody>
@@ -56,6 +57,15 @@
             <td>
               <code><%= job.seq %></code>
             </td>
+            <td>
+              <a hx-delete="<%= url_for("/jobs/enqueued/#{job.seq}", { stream_name: @stream_name, page: @page, limit: @limit }) %>"
+                 hx-target="#enqueued-poller"
+                 hx-swap="outerHTML"
+                 hx-indicator="#global-spinner"
+                 hx-confirm="Remove message #<%= job.seq %> from <%= h(@stream_name) %>?"
+                 title="Remove message"
+                 class="btn btn-danger btn-sm">&#x2715;</a>
+            </td>
           </tr>
           <% end -%>
         </tbody>