cosmonats 0.7.0 → 0.9.0

data/lib/cosmo/web/views/overview/_workers.erb

@@ -0,0 +1,25 @@
+<div class="card-head">
+  <div class="card-title">Workers</div>
+  <div class="mono mute" style="font-size:11px"><%= @busy %> / <%= @threads %> busy</div>
+</div>
+<% if @processes.empty? -%>
+  <div class="empty"><b>No worker processes</b>Start one with <code>cosmo -C config/cosmo.yml</code>.</div>
+<% else -%>
+  <div class="cells">
+    <% @cells.each do |stream| -%>
+      <i<% if stream %> style="background:<%= hue(stream) %>" title="<%= h(stream) %>"<% end %>></i>
+    <% end -%>
+  </div>
+  <% if @hidden.positive? -%><div class="small-mono">+<%= @hidden %> more threads</div><% end -%>
+  <div class="procs">
+    <% @processes.each do |process| -%>
+      <div class="proc">
+        <div class="ellip" title="<%= h(process[:cmdline]) %>"><%= h(process[:hostname]) %> · pid <%= h(process[:pid]) %></div>
+        <span class="mute"><%= h(process[:busy]) %>/<%= h(process[:concurrency]) %></span>
+        <span class="faint"><%= process[:rss] ? format_bytes(process[:rss].to_i * 1024) : '—' %></span>
+        <span style="color:<%= { 'running' => 'var(--ok)', 'quiet' => 'var(--warn)', 'stopping' => 'var(--err)' }.fetch(process[:state].to_s, 'var(--mute)') %>"><%= h(process[:state]) %></span>
+      </div>
+    <% end -%>
+  </div>
+  <a class="small-mono" href="<%= url_for('/processes') %>">All <%= @total %> process(es) &rarr;</a>
+<% end -%>

data/lib/cosmo/web/views/overview/index.erb

@@ -0,0 +1,28 @@
+<div hx-get="<%= url_for('/_kpis') %>" hx-trigger="load, cosmo:poll" data-poll="<%= poll_interval %>" hx-swap="innerHTML">
+  <div class="kpis"><div class="kpi"><div class="loading">Loading statistics&hellip;</div></div></div>
+</div>
+
+<div class="row-wrap">
+  <% if @metrics -%>
+    <section class="card card-pad grow2" id="throughput" hx-get="<%= url_for('/_throughput') %>" hx-trigger="load" hx-swap="outerHTML">
+      <div class="card-head"><div class="card-title">Throughput</div></div>
+      <div class="loading">Loading metrics&hellip;</div>
+    </section>
+  <% end -%>
+  <section class="card card-pad <%= @metrics ? 'grow1' : 'grow2' %>"
+           hx-get="<%= url_for('/_workers') %>" hx-trigger="load, cosmo:poll" data-poll="<%= poll_interval %>" hx-swap="innerHTML">
+    <div class="card-head"><div class="card-title">Workers</div></div>
+    <div class="loading">Loading workers&hellip;</div>
+  </section>
+</div>
+
+<div class="row-wrap">
+  <section class="card grow2" hx-get="<%= url_for('/jobs/_queues', compact: 1) %>" hx-trigger="load, cosmo:poll" data-poll="<%= poll_interval %>" hx-swap="innerHTML">
+    <div class="card-head bar"><div class="card-title">Queues</div></div>
+    <div class="loading">Loading queues&hellip;</div>
+  </section>
+  <section class="card grow1" hx-get="<%= url_for('/_running') %>" hx-trigger="load, cosmo:poll" data-poll="<%= poll_interval %>" hx-swap="innerHTML">
+    <div class="card-head bar ruled"><div class="card-title">Running now</div></div>
+    <div class="loading">Loading jobs&hellip;</div>
+  </section>
+</div>

data/lib/cosmo/web/views/processes/_table.erb

@@ -0,0 +1,67 @@
+<% states = { 'running' => 'var(--ok)', 'quiet' => 'var(--warn)', 'stopping' => 'var(--err)' } -%>
+<% kinds = { 'jobs' => 'Jobs', 'streams' => 'Streams', 'jobs+streams' => 'Jobs & Streams', 'none' => 'No subscriptions' } -%>
+<section class="card" id="processes"
+  <% if @polling -%>
+         hx-get="<%= url_for('/processes', @query.merge(page: @page)) %>" hx-trigger="cosmo:poll" data-poll="<%= poll_interval %>" hx-swap="outerHTML"
+  <% end -%>>
+  <div class="card-head bar ruled">
+    <div class="card-title"><%= @total %> process(es) alive</div>
+    <% if @polling -%>
+      <button type="button" class="btn quiet" hx-get="<%= url_for('/processes', @query.merge(page: @page, poll: 0)) %>"
+              hx-target="#processes" hx-swap="outerHTML" hx-indicator="#global-spinner">⏸ Stop polling</button>
+    <% else -%>
+      <button type="button" class="btn" hx-get="<%= url_for('/processes', @query.except(:poll).merge(page: @page)) %>"
+              hx-target="#processes" hx-swap="outerHTML" hx-indicator="#global-spinner">▶ Live poll</button>
+    <% end -%>
+  </div>
+
+  <% if @groups.empty? -%>
+    <div class="empty"><b>No worker processes</b>Start one with <code>cosmo -C config/cosmo.yml</code>.</div>
+  <% else -%>
+    <div class="scroll"><div style="min-width:1040px">
+      <div class="thead t-procs">
+        <span>Process</span><span>Command</span><span>Subscriptions</span><span class="r">Busy</span><span>State</span>
+        <span class="r">RTT</span><span class="r">Reconn.</span><span class="r">Memory</span><span class="r">Uptime</span><span class="r">Last beat</span>
+      </div>
+      <% @groups.each do |kind, group| -%>
+        <div class="trow group t-procs"><span class="full"><%= h(kinds.fetch(kind, kind)) %> (<%= group.size %>)</span></div>
+        <% group.each do |process| -%>
+          <% nats = Hash(process[:nats]) -%>
+          <% cmdline = process[:cmdline].to_s -%>
+          <div class="trow hover t-procs">
+            <div class="stack">
+              <span class="strong ellip"><%= h(process[:hostname]) %>:<%= h(process[:pid]) %></span>
+              <span class="small-mono ellip"><%= h(nats[:ip] || 'unknown IP') %><% if nats[:name] -%> · <%= h(nats[:name]) %><% end -%></span>
+            </div>
+            <div class="stack">
+              <% if cmdline.length > 32 -%>
+                <details>
+                  <summary class="mono ellip" style="font-size:11.5px" title="<%= h(cmdline) %>"><%= h(cmdline) %></summary>
+                  <pre class="code-block"><%= h(cmdline) %></pre>
+                </details>
+              <% else -%>
+                <span class="mono" style="font-size:11.5px"><%= h(cmdline) %></span>
+              <% end -%>
+              <span class="small-mono">cosmo <%= h(process[:version]) %> · <%= h(process[:ruby]) %><% if process[:http_port] -%> · http :<%= h(process[:http_port]) %><% end -%></span>
+            </div>
+            <div class="stack">
+              <% Hash(process[:subscriptions]).each do |type, names| -%>
+                <% next if Array(names).empty? -%>
+                <span class="mono ellip" style="font-size:11.5px" title="<%= h(Array(names).join(', ')) %>"><% if kind == 'jobs+streams' -%><span class="faint"><%= h(type) %>:</span> <% end -%><%= h(Array(names).join(', ')) %></span>
+              <% end -%>
+            </div>
+            <span class="mono r"><%= h(process[:busy]) %>/<%= h(process[:concurrency]) %></span>
+            <span class="pill" style="--tone:<%= states.fetch(process[:state].to_s, 'var(--acc)') %>"><%= h(process[:state]) %></span>
+            <span class="mono mute r"><%= nats[:rtt] ? "#{h(nats[:rtt])}ms" : '—' %></span>
+            <span class="mono mute r"><%= h(nats[:reconnects].to_i) %></span>
+            <span class="mono mute r"><%= process[:rss] ? format_bytes(process[:rss].to_i * 1024) : '—' %></span>
+            <span class="mono mute r"><%= elapsed(process[:started_at]) %></span>
+            <span class="mono mute r"><%= elapsed(process[:beat_at]) %> ago</span>
+          </div>
+        <% end -%>
+      <% end -%>
+    </div></div>
+  <% end -%>
+
+  <%= render('shared/_pager', { path: '/processes', query: @query, page: @page, total_pages: @total_pages, target: '#processes' }) %>
+</section>

data/lib/cosmo/web/views/processes/index.erb

@@ -0,0 +1 @@
+<%= @table %>

data/lib/cosmo/web/views/shared/_pager.erb

@@ -0,0 +1,22 @@
+<% if @total_pages.to_i > 1 -%>
+  <% link = ->(page) { url_for(@path, Hash(@query).merge(page: page)) } -%>
+  <nav class="pager">
+    <% if @page > 1 -%>
+      <a class="btn quiet" hx-get="<%= link.(@page - 1) %>" hx-target="<%= @target %>" hx-swap="outerHTML" hx-indicator="#global-spinner">&larr; Prev</a>
+    <% else -%>
+      <span class="btn quiet disabled">&larr; Prev</span>
+    <% end -%>
+    <% pages(@page, @total_pages).each do |p| -%>
+      <% if p == :gap -%>
+        <span class="gap">&hellip;</span>
+      <% else -%>
+        <a class="btn<%= ' cur' if p == @page %>" hx-get="<%= link.(p) %>" hx-target="<%= @target %>" hx-swap="outerHTML" hx-indicator="#global-spinner"><%= p %></a>
+      <% end -%>
+    <% end -%>
+    <% if @page < @total_pages -%>
+      <a class="btn quiet" hx-get="<%= link.(@page + 1) %>" hx-target="<%= @target %>" hx-swap="outerHTML" hx-indicator="#global-spinner">Next &rarr;</a>
+    <% else -%>
+      <span class="btn quiet disabled">Next &rarr;</span>
+    <% end -%>
+  </nav>
+<% end -%>

data/lib/cosmo/web/views/shared/unavailable.erb

@@ -0,0 +1,7 @@
+<section class="card" data-reload="<%= @retry %>">
+  <div class="empty">
+    <b>NATS is unreachable</b>
+    <span>Nothing was fetched, so this page shows no data. It tries again every <%= @retry %> seconds.</span>
+    <a class="btn" href="<%= h(@request.fullpath) %>">Retry now</a>
+  </div>
+</section>

data/lib/cosmo/web/views/streams/_info.erb

@@ -1,92 +1,34 @@
-<div class="cards-container">
-  <article class="stat-card">
-    <h3><%= format_numbers(@state.messages) %></h3>
-    <p>Messages</p>
-  </article>
-  <article class="stat-card">
-    <h3><%= format_bytes(@state.bytes) %></h3>
-    <p>Size</p>
-  </article>
-  <article class="stat-card">
-    <h3><%= @state.consumer_count %></h3>
-    <p>Consumers</p>
-  </article>
-  <article class="stat-card">
-    <h3><%= format_numbers(@state.first_seq) + " &rarr; " + format_numbers(@state.last_seq) %></h3>
-    <p>Sequence Range</p>
-  </article>
+<div>
+  <a class="small-mono" href="<%= url_for('/streams') %>">&larr; Back to streams</a>
 </div>
 
-<%= render("streams/_pause_banner", { name: @name, paused: @paused }) %>
-
-<div class="nav">
-  <a href="<%= url_for('/streams') %>"
-     hx-get="<%= url_for('/streams') %>"
-     hx-target="#content"
-     hx-push-url="true"
-     class="btn">Back to Streams</a>
+<div class="kpis">
+  <div class="kpi" style="--hue:var(--c1)"><div class="kpi-label"><i></i>Messages</div><div class="kpi-val"><b><%= format_numbers(@state.messages) %></b></div></div>
+  <div class="kpi" style="--hue:var(--c2)"><div class="kpi-label"><i></i>Size</div><div class="kpi-val"><b><%= format_bytes(@state.bytes) %></b></div></div>
+  <div class="kpi" style="--hue:var(--c3)"><div class="kpi-label"><i></i>Consumers</div><div class="kpi-val"><b><%= @state.consumer_count %></b></div></div>
+  <div class="kpi" style="--hue:var(--c4)"><div class="kpi-label"><i></i>Sequence range</div><div class="kpi-val"><b style="font-size:22px"><%= format_numbers(@state.first_seq) %> → <%= format_numbers(@state.last_seq) %></b></div></div>
 </div>
 
-<h2>Configuration</h2>
-<div class="table-container">
-  <table>
-    <tbody>
-      <tr>
-        <th>Name</th>
-        <td>
-          <code><%= h(@name) %></code>
-        </td>
-      </tr>
-      <tr>
-        <th>Retention</th>
-        <td>
-          <code><%= h(@config.retention.to_s) %></code>
-        </td>
-      </tr>
-      <tr>
-        <th>Storage</th>
-        <td>
-          <code><%= h(@config.storage.to_s) %></code>
-        </td>
-      </tr>
-      <tr>
-        <th>Discard Policy</th>
-        <td>
-          <code><%= h(@config.discard.to_s) %></code>
-        </td>
-      </tr>
-      <tr>
-        <th>Max Consumers</th>
-        <td>
-          <code><%= @config.max_consumers || "Unlimited" %></code>
-        </td>
-      </tr>
-      <tr>
-        <th>Max Messages</th>
-        <td>
-          <code><%= @config.max_msgs || "Unlimited" %></code>
-        </td>
-      </tr>
-      <tr>
-        <th>Max Bytes</th>
-        <td>
-          <code><%= @config.max_bytes&.positive? ? format_bytes(@config.max_bytes) : "Unlimited" %></code>
-        </td>
-      </tr>
-      <tr>
-        <th>Max Age</th>
-        <td>
-          <code><%= @config.max_age&.positive? ? "#{@config.max_age}ns" : "Unlimited" %></code>
-        </td>
-      </tr>
-    </tbody>
-  </table>
-</div>
+<%= render('streams/_pause_banner', { name: @name, paused: @paused }) %>
 
-<h2>Subjects</h2>
-<div class="subjects" style="margin-top: var(--space-2x);">
-  <% Array(@config.subjects).each do |sub| -%>
-    <div class="subject-tag" style="padding: var(--space) var(--space-2x); font-size: 1rem;"><%= h(sub) %></div>
+<section class="card">
+  <div class="card-head bar ruled"><div class="card-title">Configuration</div></div>
+  <% rows = {
+       'Name' => @name,
+       'Retention' => @config.retention,
+       'Storage' => @config.storage,
+       'Replicas' => @config.num_replicas || 1,
+       'Discard policy' => @config.discard,
+       'Max consumers' => @config.max_consumers.to_i.positive? ? @config.max_consumers : 'Unlimited',
+       'Max messages' => @config.max_msgs.to_i.positive? ? format_numbers(@config.max_msgs) : 'Unlimited',
+       'Max bytes' => @config.max_bytes.to_i.positive? ? format_bytes(@config.max_bytes) : 'Unlimited',
+       'Max age' => @config.max_age.to_i.positive? ? "#{format_span(@config.max_age.to_i / 1_000_000_000)} (#{@config.max_age}ns)" : 'Unlimited'
+     } -%>
+  <% rows.each do |label, value| -%>
+    <div class="trow t-kv"><span class="eyebrow"><%= label %></span><code><%= h(value) %></code></div>
   <% end -%>
-</div>
-
+  <div class="trow t-kv">
+    <span class="eyebrow">Subjects</span>
+    <div class="subjects"><% Array(@config.subjects).each do |subject| -%><span class="subject"><%= h(subject) %></span><% end -%></div>
+  </div>
+</section>

data/lib/cosmo/web/views/streams/_list.erb

@@ -0,0 +1,7 @@
+<% if @streams.empty? -%>
+  <section class="card"><div class="empty"><b>No streams</b>Run <code>cosmo --setup</code> to create them.</div></section>
+<% else -%>
+  <% @streams.each do |stream| -%>
+    <%= render('streams/_stream', { stream: }) %>
+  <% end -%>
+<% end -%>

data/lib/cosmo/web/views/streams/_pause_banner.erb

@@ -1,17 +1,9 @@
-<% if @paused -%>
-  <div id="pause-banner-<%= h(@name) %>" class="alert alert-warning" style="display:flex; align-items:center; justify-content:space-between; flex-wrap:wrap; gap:var(--space-2x);">
-    <span>&#9208; <strong>Stream is paused</strong> — consumers are not fetching new messages.</span>
-    <a hx-patch="<%= url_for('/streams/unpause', { name: u(@name), banner: 1 }) %>"
-       hx-target="#pause-banner-<%= h(@name) %>"
-       hx-swap="outerHTML"
-       class="btn btn-success">&#9654; Resume Stream</a>
+<section class="card banner" style="<%= 'border-color:color-mix(in oklch,var(--warn) 45%,var(--line))' if @paused %>">
+  <div class="stack">
+    <span class="strong"><%= @paused ? '⏸ Stream is paused' : '▶ Stream is active' %></span>
+    <span class="mute"><%= @paused ? 'Consumers are not fetching new messages.' : 'Consumers are fetching new messages.' %></span>
   </div>
-<% else -%>
-  <div id="pause-banner-<%= h(@name) %>" class="alert alert-success" style="display:flex; align-items:center; justify-content:space-between; flex-wrap:wrap; gap:var(--space-2x);">
-    <span>&#9654; <strong>Stream is active</strong> — consumers are fetching new messages.</span>
-    <a hx-patch="<%= url_for('/streams/pause', { name: u(@name), banner: 1 }) %>"
-       hx-target="#pause-banner-<%= h(@name) %>"
-       hx-swap="outerHTML"
-       class="btn btn-warning">&#9208; Pause Stream</a>
-  </div>
-<% end -%>
+  <button type="button" class="btn<%= ' quiet' unless @paused %>"
+          hx-patch="<%= url_for(@paused ? '/streams/unpause' : '/streams/pause', { name: u(@name), view: 'banner' }) %>"
+          hx-target="closest section" hx-swap="outerHTML" hx-indicator="#global-spinner"><%= @paused ? '▶ Resume stream' : '⏸ Pause stream' %></button>
+</section>

data/lib/cosmo/web/views/streams/_stream.erb

@@ -0,0 +1,52 @@
+<% s = @stream -%>
+<% state = s[:state] -%>
+<% config = s[:config] -%>
+<% max_age = config.max_age.to_i -%>
+<section class="card">
+  <div class="stream-head">
+    <div class="stack" style="gap:6px">
+      <a class="stream-name" style="--hue:<%= s[:paused] ? 'var(--warn)' : hue(s[:name]) %>"
+         href="<%= url_for('/streams/info', name: u(s[:name])) %>"><i></i><%= h(s[:name]) %></a>
+      <div class="mono mute ellip" style="font-size:11.5px" title="<%= h(Array(config.subjects).join(', ')) %>"><%= h(Array(config.subjects).join('  ·  ')) %></div>
+    </div>
+    <div class="stat"><span class="eyebrow">Messages</span><b><%= compact_number(state.messages) %></b></div>
+    <div class="stat"><span class="eyebrow">Size</span><b><%= format_bytes(state.bytes) %></b></div>
+    <div class="stat"><span class="eyebrow">Retention</span><b><%= h(config.retention) %><%= " · #{format_span(max_age / 1_000_000_000)}" if max_age.positive? %></b></div>
+    <div class="stat"><span class="eyebrow">Replicas</span><b>R<%= h(config.num_replicas || 1) %></b></div>
+    <div class="stat"><span class="eyebrow">Sequence</span><b class="ellip"><%= compact_number(state.first_seq) %> → <%= compact_number(state.last_seq) %></b></div>
+  </div>
+  <div class="stream-tools">
+    <% if s[:paused] -%><span class="paused-tag">paused · consumers are not fetching</span><% end -%>
+    <span class="spacer"></span>
+    <a class="btn quiet" href="<%= url_for('/streams/info', name: u(s[:name])) %>">Details</a>
+    <button type="button" class="btn<%= ' quiet' unless s[:paused] %>"
+            hx-patch="<%= url_for(s[:paused] ? '/streams/unpause' : '/streams/pause', name: u(s[:name])) %>"
+            hx-target="closest section" hx-swap="outerHTML" hx-indicator="#global-spinner"><%= s[:paused] ? '▶ Resume' : '⏸ Pause' %></button>
+  </div>
+  <% if s[:consumers].empty? -%>
+    <div class="thead top tint"><span>No consumers</span></div>
+  <% else -%>
+    <div class="scroll"><div style="min-width:760px">
+      <div class="thead top tint t-cons"><span>Consumer</span><span class="r">Pending</span><span class="r">Ack pending</span><span class="r">Ack wait</span><span class="r">Redeliv.</span><span>Lag</span></div>
+      <% s[:consumers].each do |consumer| -%>
+        <% cfg = Hash(consumer['config']) -%>
+        <% pending = consumer['num_pending'].to_i -%>
+        <% redelivered = consumer['num_redelivered'].to_i -%>
+        <% ratio = pending.fdiv([state.messages.to_i, 1].max).clamp(0, 1) -%>
+        <% tone = ratio > 0.7 ? 'var(--err)' : ratio > 0.3 ? 'var(--warn)' : 'var(--ok)' -%>
+        <% filters = Array(cfg['filter_subjects']).concat(Array(cfg['filter_subject'])).reject(&:empty?) -%>
+        <div class="trow t-cons">
+          <div class="stack">
+            <span class="strong ellip"><%= h(consumer['name']) %></span>
+            <span class="small-mono ellip" title="<%= h(filters.join(', ')) %>">deliver: <%= h(cfg['deliver_policy']) %><%= " · #{h(filters.join(', '))}" if filters.any? %></span>
+          </div>
+          <span class="r"><%= format_numbers(pending) %></span>
+          <span class="r mute"><%= format_numbers(consumer['num_ack_pending'].to_i) %></span>
+          <span class="r mute"><%= format_span(cfg['ack_wait'].to_i / 1_000_000_000) %></span>
+          <span class="r" style="color:<%= redelivered > 100 ? 'var(--warn)' : 'var(--mute)' %>"><%= format_numbers(redelivered) %></span>
+          <div class="lag"><div class="lag-track"><i style="width:<%= [ratio * 100, 2].max.round(1) %>%;background:<%= tone %>"></i></div><span style="color:<%= tone %>"><%= compact_number(pending) %></span></div>
+        </div>
+      <% end -%>
+    </div></div>
+  <% end -%>
+</section>

data/lib/cosmo/web/views/streams/index.erb

@@ -1,11 +1,3 @@
-<section>
-  <header></header>
-
-  <div id="content">
-    <div hx-get="<%= url_for('/streams/_table') %>"
-         hx-trigger="load, every <%= poll_interval %>s"
-         hx-swap="innerHTML">
-      <div class="alert alert-info">Loading streams…</div>
-    </div>
-  </div>
-</section>
+<div class="view" hx-get="<%= url_for('/streams') %>" hx-trigger="load, cosmo:poll" data-poll="<%= poll_interval %>" hx-swap="innerHTML">
+  <section class="card"><div class="loading">Loading streams&hellip;</div></section>
+</div>

data/lib/cosmo/web/views/streams/info.erb

@@ -1,11 +1,3 @@
-<section>
-  <header></header>
-
-  <div id="content">
-    <div hx-get="<%= url_for("/streams/_info", { name: u(@name) }) %>"
-         hx-trigger="load"
-         hx-swap="innerHTML">
-      <div class="alert alert-info">Loading stream info…</div>
-    </div>
-  </div>
-</section>
+<div class="view" hx-get="<%= url_for('/streams/info', { name: u(@name) }) %>" hx-trigger="load" hx-swap="innerHTML">
+  <section class="card"><div class="loading">Loading stream info&hellip;</div></section>
+</div>

data/lib/cosmo/web.rb

@@ -3,68 +3,102 @@
 require "rack"
 require "json"
 require "erb"
+require "zlib"
+require "cosmo/web/cache"
+require "cosmo/web/circuit"
 require "cosmo/web/context"
 require "cosmo/web/renderer"
 require "cosmo/web/controllers/application"
+require "cosmo/web/controllers/overview"
 require "cosmo/web/controllers/jobs"
+require "cosmo/web/controllers/processes"
 require "cosmo/web/controllers/streams"
-require "cosmo/web/controllers/actions"
 require "cosmo/web/controllers/crons"
 require "cosmo/web/controllers/batches"
+require "cosmo/web/controllers/metrics"
 
 module Cosmo
   class Web
     include Renderer
 
+    FONT_HEADERS = { "cache-control" => "public, max-age=31536000, immutable" }.freeze
+
     def self.call(env)
       new.call(env)
     end
 
-    def call(env) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+    # Serves the UI. An htmx refresh is shared through Cache with concurrent and recent identical ones; any other
+    # method than GET changes something, so it empties the cache. While Circuit is open, anything but an asset
+    # answers 503 at once instead of waiting on NATS.
+    def call(env) # rubocop:disable Metrics/AbcSize
       @request = Rack::Request.new(env)
+      return route if @request.path_info.start_with?("/assets/", "/favicon.ico")
+      return guarded.tap { Cache.instance.clear } unless @request.get?
+      return guarded unless @request.get_header("HTTP_HX_REQUEST") == "true"
+
+      status, headers, body = Cache.instance.fetch(@request.fullpath) { guarded }
+      [status, headers.dup, body.dup]
+    rescue *Circuit::ERRORS
+      Circuit.instance.trip!
+      unavailable
+    end
+
+    private
+
+    def route # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       path     = @request.path_info
       method   = @request.request_method
 
       response = case [method.downcase.to_sym, path]
-                 in [:get,    "/"]                      then redirect_to("/jobs")
-                 in [:get,    "/jobs"]                  then [Controllers::Jobs,    :index]
-                 in [:get,    "/jobs/scheduled"]        then [Controllers::Jobs,    :scheduled]
-                 in [:get,    "/jobs/dead"]             then [Controllers::Jobs,    :dead]
-                 in [:get,    "/jobs/busy"]             then [Controllers::Jobs,    :busy]
-                 in [:get,    "/jobs/enqueued"]         then [Controllers::Jobs,    :enqueued]
-                 in [:patch,  %r{/jobs/retry/\d+}]      then [Controllers::Jobs,    :retry]
-                 in [:delete, %r{/jobs/delete/\d+}]     then [Controllers::Jobs,    :delete]
-                 in [:delete, %r{/jobs/enqueued/\d+}]   then [Controllers::Jobs,    :delete_enqueued]
-                 in [:get,    "/jobs/_stats"]           then [Controllers::Jobs,    :_stats]
-                 in [:get,    "/jobs/_scheduled"]       then [Controllers::Jobs,    :_scheduled]
-                 in [:get,    "/jobs/_dead"]            then [Controllers::Jobs,    :_dead]
-                 in [:get,    "/jobs/_busy"]            then [Controllers::Jobs,    :_busy]
-                 in [:get,    "/jobs/_processes"]       then [Controllers::Jobs,    :_processes]
-                 in [:get,    "/jobs/_enqueued"]        then [Controllers::Jobs,    :_enqueued]
-                 in [:get,    "/streams"]               then [Controllers::Streams, :index]
-                 in [:get,    "/streams/info"]          then [Controllers::Streams, :info]
-                 in [:get,    "/streams/_table"]        then [Controllers::Streams, :_table]
-                 in [:get,    "/streams/_info"]         then [Controllers::Streams, :_info]
-                 in [:patch,  "/streams/pause"]         then [Controllers::Streams, :pause]
-                 in [:patch,  "/streams/unpause"]       then [Controllers::Streams, :unpause]
-                 in [:get,    "/crons"]                 then [Controllers::Crons,   :index]
-                 in [:get,    "/crons/_table"]          then [Controllers::Crons,   :_table]
-                 in [:delete, "/crons/delete"]          then [Controllers::Crons,   :delete]
-                 in [:post,   "/crons/run"]             then [Controllers::Crons,   :run_now]
-                 in [:get,    "/batches"]               then [Controllers::Batches, :index]
-                 in [:get,    "/batches/_table"]        then [Controllers::Batches, :_table]
-                 in [:get,    "/actions"]               then [Controllers::Actions, :index]
+                 in [:get,    "/" | ""]                 then [Controllers::Overview,  :index]
+                 in [:get,    "/_kpis"]                 then [Controllers::Overview,  :_kpis]
+                 in [:get,    "/_throughput"]           then [Controllers::Overview,  :_throughput]
+                 in [:get,    "/_workers"]              then [Controllers::Overview,  :_workers]
+                 in [:get,    "/_running"]              then [Controllers::Overview,  :_running]
+                 in [:get,    "/_badges"]               then [Controllers::Overview,  :_badges]
+                 in [:get,    "/jobs"]                  then [Controllers::Jobs,      :index]
+                 in [:get,    "/jobs/busy"]             then [Controllers::Jobs,      :busy]
+                 in [:get,    "/jobs/enqueued"]         then [Controllers::Jobs,      :enqueued]
+                 in [:get,    "/jobs/scheduled"]        then [Controllers::Jobs,      :scheduled]
+                 in [:get,    "/jobs/dead"]             then [Controllers::Jobs,      :dead]
+                 in [:get,    "/jobs/_queues"]          then [Controllers::Jobs,      :_queues]
+                 in [:patch,  %r{/jobs/retry/\d+}]      then [Controllers::Jobs,      :retry]
+                 in [:delete, %r{/jobs/delete/\d+}]     then [Controllers::Jobs,      :delete]
+                 in [:delete, %r{/jobs/enqueued/\d+}]   then [Controllers::Jobs,      :delete_enqueued]
+                 in [:get,    "/processes"]             then [Controllers::Processes, :index]
+                 in [:get,    "/streams"]               then [Controllers::Streams,   :index]
+                 in [:get,    "/streams/info"]          then [Controllers::Streams,   :info]
+                 in [:patch,  "/streams/pause"]         then [Controllers::Streams,   :pause]
+                 in [:patch,  "/streams/unpause"]       then [Controllers::Streams,   :unpause]
+                 in [:get,    "/crons"]                 then [Controllers::Crons,     :index]
+                 in [:get,    "/crons/_table"]          then [Controllers::Crons,     :_table]
+                 in [:delete, "/crons/delete"]          then [Controllers::Crons,     :delete]
+                 in [:post,   "/crons/run"]             then [Controllers::Crons,     :run_now]
+                 in [:get,    "/metrics"]               then [Controllers::Metrics,   :index]
+                 in [:get,    "/metrics/_panel"]        then [Controllers::Metrics,   :_panel]
+                 in [:get,    "/batches"]               then [Controllers::Batches,   :index]
+                 in [:get,    "/batches/_table"]        then [Controllers::Batches,   :_table]
                  in [:get,    "/assets/htmx.min.js.gz"] then serve("htmx.2.0.8.min.js.gz",
                                                                    "application/javascript",
                                                                    { "content-encoding" => "gzip" })
-                 in [:get, "/assets/app.css"]         then serve("app.css", "text/css")
-                 in [:get, "/favicon.ico"]            then no_content
+                 in [:get, "/assets/app.css"]              then serve("app.css", "text/css")
+                 in [:get, "/assets/app.js"]               then serve("app.js", "application/javascript")
+                 in [:get, "/assets/logo.svg"]             then serve("logo.svg", "image/svg+xml")
+                 in [:get, "/assets/fonts/geist.woff2"]    then serve("fonts/geist.woff2", "font/woff2", FONT_HEADERS)
+                 in [:get, "/assets/fonts/geist-mono.woff2"] then serve("fonts/geist-mono.woff2", "font/woff2", FONT_HEADERS)
+                 in [:get, "/favicon.ico"] then redirect_to("/assets/logo.svg", 301)
                  else not_found
                  end
       handler(response)
     end
 
-    private
+    def guarded
+      Circuit.instance.open? ? unavailable : route
+    end
+
+    def unavailable
+      Controllers::Application.new(@request).unavailable
+    end
 
     def handler(response)
       if response[0].is_a?(Class)

data/lib/cosmo.rb

@@ -14,6 +14,8 @@ require "cosmo/cli"
 require "cosmo/heartbeat"
 require "cosmo/engine"
 require "cosmo/api"
+require "cosmo/middleware"
+require "cosmo/services"
 require "cosmo/railtie" if defined?(Rails::Railtie)
 
 module Cosmo
@@ -22,18 +24,54 @@ module Cosmo
   # Optional, requires rack and webrick gems. Loaded only when an HTTP port is configured.
   autoload :HTTPServer, "cosmo/http_server"
 
+  # Programmatic setup, typically from an initializer:
+  #
+  #   Cosmo.configure do |config|
+  #     config.logger = Rails.logger
+  #     config.log_level = :debug
+  #     config.server_middleware { |c| c.add MyMiddleware }
+  #   end
+  #
+  # @yieldparam config [Config] the {Config} singleton
+  # @return [void]
+  def self.configure
+    yield Config.instance
+  end
+
+  # Hands +error+ to every handler in {Config#error_handlers}. A handler that raises is logged and skipped, so error
+  # reporting never breaks processing.
+  #
+  # @param error [Exception]
+  # @param context [Hash] where it happened, see {Config#error_handlers}
+  # @return [void]
+  def self.handle_error(error, context)
+    Config.error_handlers.each do |handler|
+      handler.call(error, context)
+    rescue StandardError => e
+      Logger.error "Error handler failed: #{e.class}: #{e.message}"
+    end
+  end
+
   class Error < StandardError; end
 
   class ArgumentError < Error; end
 
   class NotImplementedError < Error; end
 
+  class ConfigError < Error; end
+
   class ConfigNotFoundError < Error
     def initialize(config_file)
       super("No such file #{config_file}")
     end
   end
 
+  class SchedulingDisabledError < Error
+    def initialize
+      super("Scheduling is turned off (config.scheduled.enabled = false), so delayed jobs and crons are unavailable")
+    end
+  end
+
   class StreamNotFoundError < Error
     def initialize(stream_name)
       super("Missing stream `#{stream_name}`")