cosmonats 0.7.0 → 0.9.0

data/lib/cosmo/web/controllers/jobs.rb

@@ -7,109 +7,114 @@ module Cosmo
     module Controllers
       class Jobs < Application
         def index
-          content_for :title, "Jobs"
-          ok render("jobs/index", layout: true)
+          tab("busy")
         end
 
         def busy
-          return _busy if hx_request?
-
-          content_for :title, "Busy Jobs"
-          ok render("jobs/busy", layout: true)
+          tab("busy")
         end
 
         def enqueued
-          return _enqueued if hx_request?
-
-          content_for :title, "Enqueued Jobs"
-          stream_name, _stream_names = streams
-          ok render("jobs/enqueued", { stream_name: }, layout: true)
+          tab("enqueued")
         end
 
         def scheduled
-          return _scheduled if hx_request?
+          return not_found unless Config.scheduled.enabled
 
-          content_for :title, "Scheduled Jobs"
-          ok render("jobs/scheduled", layout: true)
+          tab("scheduled")
         end
 
         def dead
-          return _dead if hx_request?
+          return not_found unless Config.dead.enabled
+
+          tab("dead")
+        end
 
-          content_for :title, "Dead Jobs"
-          ok render("jobs/dead", layout: true)
+        def _queues
+          ok render("jobs/_queues", { queues: API::Stats.queues, compact: params["compact"].to_s == "1" })
         end
 
         def retry
+          return not_found unless Config.dead.enabled
+
           seq = path.split("/").last.to_i
-          stream = API::Stream.new("dead")
-          stream.retry(seq)
-          ok
+          API::Stream.new("dead").retry(seq)
+          ok flash("Dead job ##{seq} sent back to its queue")
         end
 
         def delete
+          return not_found unless Config.dead.enabled
+
           seq = path.split("/").last.to_i
-          stream = API::Stream.new("dead")
-          stream.delete(seq)
-          ok
+          API::Stream.new("dead").delete(seq)
+          ok flash("Dead job ##{seq} deleted")
         end
 
         def delete_enqueued
           seq = path.split("/").last.to_i
           stream_name, = streams
           API::Stream.new(stream_name).delete(seq)
-          _enqueued
+          ok panel("enqueued") + flash("Message ##{seq} removed from #{stream_name}")
+        end
+
+        private
+
+        def tab(name)
+          return ok(panel(name)) if hx_request?
+
+          content_for :title, "Jobs"
+          content_for :subtitle, "Queues, in-flight work and everything waiting its turn."
+          ok render("jobs/index", { panel: panel(name) }, layout: true)
         end
 
-        def _scheduled
-          stream = API::Stream.new("scheduled")
-          jobs = stream.messages(page: params["page"], limit: params["limit"])
-          ok render("jobs/_scheduled", { jobs: jobs, total: stream.total })
+        def panel(name)
+          render("jobs/_panel", send("#{name}_locals").merge(tab: name, counts: counts))
         end
 
-        def _dead
-          stream = API::Stream.new("dead")
-          jobs = stream.messages(page: params["page"], limit: params["limit"])
-          ok render("jobs/_dead", { jobs: jobs, total: stream.total })
+        def counts
+          counts = { busy: API::Stats.busy, enqueued: API::Stats.enqueued }
+          counts[:scheduled] = API::Stats.scheduled if Config.scheduled.enabled
+          counts[:dead] = API::Stats.dead if Config.dead.enabled
+          counts
         end
 
-        def _busy
+        def busy_locals
           busy = API::Stats::Busy.instance
           total = busy.size
           page, limit, total_pages = paginate(total, API::Stats::Busy::LIMIT)
           polling = params["poll"].to_s != "0"
-          ok render("jobs/_busy", { jobs: busy.list(page:, limit:), total:, page:, limit:, total_pages:, polling: })
-        end
-
-        def _processes
-          processes = API::Stats::Processes.instance
-          total = processes.size
-          page, limit, total_pages = paginate(total, API::Stats::Processes::LIMIT)
-          polling = params["poll"].to_s != "0"
-          ok render("jobs/_processes", { processes: processes.list(page:, limit:), total:, page:, limit:, total_pages:, polling: })
+          { jobs: busy.list(page:, limit:), total:, page:, limit:, total_pages:, polling:, query: { limit:, poll: (0 unless polling) }.compact }
         end
 
-        def _enqueued # rubocop:disable Metrics/AbcSize
+        def enqueued_locals # rubocop:disable Metrics/AbcSize
           stream_name, stream_names = streams
           limit = (params["limit"] || API::Stream::LIMIT).to_i
           page = [params["page"].to_i, 1].max
+          locals = { stream_name:, stream_names:, page:, limit:, total: 0, total_pages: 0, jobs: nil, polling: false,
+                     query: { stream_name: Rack::Utils.escape(stream_name.to_s), limit: } }
+          return locals if stream_name.to_s.empty?
 
-          unless stream_name.to_s.empty?
-            stream = API::Stream.new(stream_name)
-            total = stream.total
-            total_pages = (total.to_f / limit).ceil
-            page = page.clamp(1, [total_pages, 1].max)
-            jobs = stream.messages(page:, limit:)
-          end
+          stream = API::Stream.new(stream_name)
+          total = stream.total
+          total_pages = (total.to_f / limit).ceil
+          page = page.clamp(1, [total_pages, 1].max)
+          locals.merge(total:, total_pages:, page:, jobs: stream.messages(page:, limit:))
+        end
 
-          ok render("jobs/_enqueued", { jobs:, total:, stream_name:, stream_names:, page:, limit:, total_pages: })
+        def scheduled_locals
+          stored_locals("scheduled").merge(polling: true)
         end
 
-        def _stats
-          ok render("jobs/_stats", API::Stats.summary)
+        def dead_locals
+          stored_locals("dead").merge(polling: false)
         end
 
-        private
+        def stored_locals(name)
+          stream = API::Stream.new(name)
+          total = stream.total
+          page, limit, total_pages = paginate(total, API::Stream::LIMIT)
+          { jobs: stream.messages(page:, limit:), total:, page:, limit:, total_pages:, query: { limit: } }
+        end
 
         def paginate(total, default_limit)
           limit = (params["limit"] || default_limit).to_i

data/lib/cosmo/web/controllers/metrics.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "cosmo/web/controllers/application"
+require "cosmo/web/chart"
+
+module Cosmo
+  class Web
+    module Controllers
+      class Metrics < Application
+        RANGES = { 1 => "Today", 7 => "7 days", 30 => "30 days" }.freeze
+
+        def index
+          return not_found unless Config.metrics.enabled
+
+          content_for :title, "Metrics"
+          content_for :subtitle, "Runs, failures, execution and wait time per job class."
+          ok render("metrics/index", { days:, ranges: RANGES }, layout: true)
+        end
+
+        # The chart and the table together, so the charted job stays highlighted across refreshes.
+        def _panel
+          return not_found unless Config.metrics.enabled
+
+          rows = metrics.summary(days:)
+          job = charted(rows)
+          chart = Chart.new(metrics.daily(job, days:)) if job && days > 1
+          ok render("metrics/_panel", { rows:, job:, days:, chart: })
+        end
+
+        private
+
+        # The requested job while it has rows in the period, the busiest one otherwise.
+        def charted(rows)
+          job = params["job"].to_s
+          rows.any? { _1[:job] == job } ? job : rows.first&.dig(:job)
+        end
+
+        def days
+          value = params["days"].to_i
+          RANGES.key?(value) ? value : 7
+        end
+
+        def metrics
+          API::Stats::Metrics.instance
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/web/controllers/overview.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "cosmo/web/controllers/application"
+require "cosmo/web/throughput"
+
+module Cosmo
+  class Web
+    module Controllers
+      class Overview < Application
+        RANGES = [7, 30].freeze
+        CELLS = 256
+        RUNNING = 9
+        PROCESSES = 5
+
+        def index
+          content_for :title, "Overview"
+          content_for :subtitle, "Everything beamed through NATS, live."
+          ok render("overview/index", { metrics: Config.metrics.enabled }, layout: true)
+        end
+
+        def _kpis
+          ok render("overview/_kpis", API::Stats.summary)
+        end
+
+        def _throughput
+          return not_found unless Config.metrics.enabled
+
+          days = RANGES.include?(params["days"].to_i) ? params["days"].to_i : RANGES.first
+          chart = Throughput.new(API::Stats::Metrics.instance.daily(days:))
+          ok render("overview/_throughput", { chart:, days:, ranges: RANGES })
+        end
+
+        def _workers # rubocop:disable Metrics/AbcSize
+          processes = API::Stats::Processes.instance.list(limit: API::Stats::Processes.instance.size.clamp(1, nil))
+          busy = API::Stats::Busy.instance.all.group_by { _1[:worker] }
+          cells = processes.flat_map { worker_cells(_1, busy) }
+          ok render("overview/_workers", { processes: processes.first(PROCESSES), total: processes.size,
+                                           cells: cells.first(CELLS), hidden: [cells.size - CELLS, 0].max,
+                                           busy: cells.count(&:itself), threads: cells.size })
+        end
+
+        def _running
+          jobs = API::Stats::Busy.instance.all.sort_by { _1[:started_at].to_i }.first(RUNNING)
+          jobs = jobs.map { _1.merge(data: Utils::Json.parse(_1[:data]) || {}) }
+          ok render("overview/_running", { jobs:, total: API::Stats::Busy.instance.size })
+        end
+
+        def _badges
+          badges = { jobs: API::Stats.enqueued, streams: visible_streams.size }
+          badges[:crons] = API::Cron.instance.all.size if Config.scheduled.enabled
+          badges[:batches] = API::Batch.kv.keys("*.meta", limit: nil).size
+          ok render("overview/_badges", { badges: })
+        end
+
+        private
+
+        # A process's threads, each the stream of the job it runs or nil while idle.
+        def worker_cells(process, busy)
+          streams = Array(busy[process[:identity]]).map { _1[:stream] }
+          idle = [process[:concurrency].to_i - streams.size, 0].max
+          streams + Array.new(idle)
+        end
+
+        def visible_streams
+          API::Stream.all.reject { _1.name.start_with?("KV_", "_cosmo") }
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/web/controllers/processes.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "cosmo/web/controllers/application"
+
+module Cosmo
+  class Web
+    module Controllers
+      class Processes < Application
+        def index
+          return ok(table) if hx_request?
+
+          content_for :title, "Processes"
+          content_for :subtitle, "Every worker process heartbeating into NATS."
+          ok render("processes/index", { table: table }, layout: true)
+        end
+
+        private
+
+        def table # rubocop:disable Metrics/AbcSize
+          processes = API::Stats::Processes.instance
+          total = processes.size
+          limit = (params["limit"] || API::Stats::Processes::LIMIT).to_i
+          total_pages = (total.to_f / limit).ceil
+          page = params["page"].to_i.clamp(1, [total_pages, 1].max)
+          polling = params["poll"].to_s != "0"
+          groups = processes.list(page:, limit:).chunk { API::Stats::Processes.kind(_1) }.to_a
+          render("processes/_table", { groups:, total:, page:, limit:, total_pages:, polling:,
+                                       query: { limit:, poll: (0 unless polling) }.compact })
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/web/controllers/streams.rb

@@ -7,62 +7,60 @@ module Cosmo
     module Controllers
       class Streams < Application
         def index
-          return _table if hx_request?
+          return ok(list) if hx_request?
 
           content_for :title, "Streams"
+          content_for :subtitle, "JetStream streams and the consumers reading them."
           ok render("streams/index", layout: true)
         end
 
         def info
-          name = Rack::Utils.unescape(@request.params["name"])
-          return _info if hx_request?
+          return ok(render("streams/_info", info_locals(API::Stream.new(name)))) if hx_request?
 
           content_for :title, "Streams"
-          ok render("streams/info", { name: name }, layout: true)
+          content_for :subtitle, "Configuration of #{name}."
+          ok render("streams/info", { name: }, layout: true)
         end
 
         def pause
-          name = Rack::Utils.unescape(@request.params["name"])
           stream = API::Stream.new(name)
           stream.pause!
-          return ok render("streams/_pause_banner", banner_locals(stream)) if @request.params["banner"]
-
-          ok render("streams/_stream_row", { stream: row_locals(stream) })
+          toggled(stream, "paused")
         end
 
         def unpause
-          name = Rack::Utils.unescape(@request.params["name"])
           stream = API::Stream.new(name)
           stream.unpause!
-          return ok render("streams/_pause_banner", banner_locals(stream)) if @request.params["banner"]
-
-          ok render("streams/_stream_row", { stream: row_locals(stream) })
+          toggled(stream, "resumed")
         end
 
-        def _table
-          streams = API::Stream.all.reject { |s| s.name.start_with?("KV_") || s.name.start_with?("_cosmo") }.map { row_locals(_1) }
-          ok render("streams/_table", { streams: streams })
+        private
+
+        def name
+          Rack::Utils.unescape(params["name"].to_s)
         end
 
-        def _info
-          name = Rack::Utils.unescape(@request.params["name"])
-          stream = API::Stream.new(name)
-          ok render("streams/_info", stream.info.merge(name:, paused: stream.paused?))
+        def list
+          streams = API::Stream.all.reject { _1.name.start_with?("KV_", "_cosmo") }.map { card(_1) }
+          render("streams/_list", { streams: })
         end
 
-        private
+        def toggled(stream, verb)
+          body = case params["view"] || (params["banner"] && "banner")
+                 when "banner" then render("streams/_pause_banner", { name: stream.name, paused: stream.paused? })
+                 when "queue" then render("jobs/_queue_row", { queue: API::Stats.queue(stream), compact: false })
+                 else render("streams/_stream", { stream: card(stream) })
+                 end
+          ok body + flash("#{stream.name} #{verb}")
+        end
 
-        def row_locals(stream)
+        def card(stream)
           state, config = stream.info.values
-          { name: stream.name, messages: state.messages, bytes: state.bytes,
-            first_seq: state.first_seq, last_seq: state.last_seq,
-            consumer_count: state.consumer_count,
-            subjects: config.subjects,
-            paused: stream.paused? }
+          { name: stream.name, state:, config:, consumers: stream.consumers, paused: stream.paused? }
         end
 
-        def banner_locals(stream)
-          { name: stream.name, paused: stream.paused? }
+        def info_locals(stream)
+          stream.info.merge(name: stream.name, paused: stream.paused?)
         end
       end
     end

data/lib/cosmo/web/helpers/application.rb

@@ -42,6 +42,16 @@ module Cosmo
           value.to_s
         end
 
+        # @param millis [Numeric, nil]
+        # @return [String] e.g. "340 ms", "1.25 s", "2m 3s", or "—" without a value
+        def duration(millis)
+          return "—" unless millis
+          return "#{millis.round} ms" if millis < 1_000
+          return "#{(millis / 1_000.0).round(2)} s" if millis < 60_000
+
+          "#{(millis / 60_000).floor}m #{((millis % 60_000) / 1_000).round}s"
+        end
+
         def elapsed(value)
           elapsed = Time.now.to_i - value.to_i
 
@@ -68,6 +78,63 @@ module Cosmo
           "#{(diff / 86_400).to_i}d"
         end
 
+        # @return [String] e.g. "940", "18.9k", "1.28M"
+        def compact_number(num)
+          n = num.to_i
+          return format("%.2fM", n / 1e6).sub(/\.?0+M\z/, "M") if n >= 1_000_000
+          return format("%.1fk", n / 1e3).sub(/\.0k\z/, "k") if n >= 10_000
+
+          format_numbers(n)
+        end
+
+        # @param seconds [Numeric, nil]
+        # @return [String] e.g. "45s", "30m", "7d", or "—" without a value
+        def format_span(seconds)
+          s = seconds.to_i
+          return "—" unless s.positive?
+
+          { 86_400 => "d", 3_600 => "h", 60 => "m" }.each { |size, unit| return "#{(s / size.to_f).round(1).to_s.delete_suffix(".0")}#{unit}" if s >= size }
+          "#{s}s"
+        end
+
+        # A stable chart color per name, one of the palette's five hues: job streams take them in name order, so the
+        # first five never share one, anything else by a hash of its name.
+        def hue(name)
+          @hues ||= API::Stream.jobs.map(&:name).sort.each_with_index.to_h
+          index = @hues.fetch(name.to_s) { Zlib.crc32(name.to_s) }
+          "var(--c#{(index % 5) + 1})"
+        end
+
+        # @return [Array<Hash>] the sidebar entries, +:key+, +:label+ and +:path+, for the enabled pages
+        def nav_items
+          items = [%w[overview Overview /], %w[jobs Jobs /jobs], %w[streams Streams /streams]]
+          items << %w[crons Crons /crons] if Config.scheduled.enabled
+          items << %w[batches Batches /batches]
+          items << %w[metrics Metrics /metrics] if Config.metrics.enabled
+          items.map { |key, label, path| { key:, label:, path: } }
+        end
+
+        # @return [String] the nav key of the page being rendered
+        def current_section
+          path = @request.path_info
+          return "overview" if path.empty? || path == "/" || path.start_with?("/processes")
+
+          path.split("/")[1].to_s
+        end
+
+        # @return [Hash] the connected NATS server's +:url+, +:name+ and +:version+, empty while unknown
+        def nats_server
+          return {} if Web::Circuit.instance.open?
+
+          connection = Client.instance.nc
+          info = connection.server_info || {}
+          uri = connection.connected_server
+          name = info[:server_name] unless info[:server_name] == info[:server_id]
+          { url: uri && "#{uri.host}:#{uri.port}", name:, version: info[:version] }.compact
+        rescue StandardError
+          {}
+        end
+
         def h(value)
           Rack::Utils.escape_html(value.to_s)
         end

data/lib/cosmo/web/renderer.rb

@@ -6,6 +6,22 @@ module Cosmo
       ASSETS_ROOT = File.expand_path("assets", __dir__).freeze
       VIEWS_ROOT  = File.expand_path("views",  __dir__).freeze
 
+      @templates = {}
+      @lock = Mutex.new
+
+      # Compiles a view into a Context method on its first render, so later renders skip reading and parsing it.
+      #
+      # @param path [String] the view, relative to VIEWS_ROOT and without +.erb+, e.g. +"jobs/_busy"+
+      # @return [String] the name of the Context method that renders it
+      def self.template(path)
+        @lock.synchronize do
+          @templates[path] ||= "template#{@templates.size}".tap do |name|
+            source = File.read(File.join(VIEWS_ROOT, "#{path}.erb"))
+            ERB.new(source, trim_mode: "-").def_method(Context, name, "#{path}.erb")
+          end
+        end
+      end
+
       def redirect_to(url, status = 302)
         [status, { "location" => url_for(url) }, []]
       end
@@ -14,7 +30,7 @@ module Cosmo
         path = File.join(ASSETS_ROOT, filename)
         return not_found unless File.exist?(path)
 
-        respond_with 200, body: File.read(path), headers: { "content-type" => content_type }.merge(headers || {})
+        respond_with 200, body: File.binread(path), headers: { "content-type" => content_type }.merge(headers || {})
       end
 
       def ok(body = "")
@@ -27,7 +43,7 @@ module Cosmo
       end
 
       def not_found
-        body = "<div class='alert alert-danger'>404 — Not Found</div>"
+        body = "<div class=\"empty\"><b>404</b>Not found</div>"
         headers = { "content-type" => "text/html; charset=utf-8" }
         respond_with 404, body:, headers:
       end
@@ -48,10 +64,7 @@ module Cosmo
       end
 
       def erb(path, locals, content_for = nil)
-        path = File.join(VIEWS_ROOT, "#{path}.erb")
-        erb  = ERB.new(File.read(path), trim_mode: "-")
-        context = Context.new(locals, content_for)
-        erb.result(context.binding)
+        Context.new(locals, content_for).public_send(Renderer.template(path))
       end
     end
   end

data/lib/cosmo/web/throughput.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class Web
+    # Geometry for the Overview's throughput chart: an area line of the successful runs per day and a bar per day for
+    # the failed ones, drawn into a viewBox the page stretches to its width, axes and hover targets laid out in percent.
+    class Throughput
+      WIDTH = 1000
+      HEIGHT = 220
+      PLOT = 214
+      FAILED = 54
+      BAR = 22
+      TICKS = 3
+      LABELS = 5
+
+      # @param series [Array<Hash>] +{ date:, count:, failed:, exec_ms: }+ per day, oldest first
+      def initialize(series)
+        @series = series
+        @top = top(series.map { _1[:count] }.max.to_i)
+        @failed_top = [series.map { _1[:failed] }.max.to_i, 1].max
+      end
+
+      # @return [Boolean]
+      def empty?
+        @series.all? { _1[:count].zero? && _1[:failed].zero? }
+      end
+
+      # @return [String] the path of the runs line
+      def line
+        @series.each_with_index.map { |day, index| "#{index.zero? ? "M" : "L"}#{x(index)} #{y(day[:count])}" }.join(" ")
+      end
+
+      # @return [String] the path of the area under the runs line
+      def area
+        "#{line} L#{x(@series.size - 1)} #{HEIGHT} L#{x(0)} #{HEIGHT} Z"
+      end
+
+      # @return [String] the path of the failure bars, one per day with failures
+      def failures # rubocop:disable Metrics/AbcSize
+        width = ((WIDTH / [@series.size, 1].max.to_f) * 0.42).clamp(2, BAR).round(1)
+        @series.each_with_index.filter_map do |day, index|
+          next if day[:failed].zero?
+
+          height = [(day[:failed] / @failed_top.to_f) * FAILED, 1].max.round(1)
+          "M#{(x(index) - (width / 2)).round(1)} #{HEIGHT} v#{-height} h#{width} v#{height} Z"
+        end.join(" ")
+      end
+
+      # @return [String] the path of the horizontal grid lines
+      def grid
+        (0..TICKS).map { "M0 #{(HEIGHT - (PLOT * _1 / TICKS.to_f)).round(1)} H#{WIDTH}" }.join(" ")
+      end
+
+      # @return [Array<Integer>] the y axis values, top down
+      def ticks
+        TICKS.downto(0).map { (@top * _1 / TICKS.to_f).round }
+      end
+
+      # @return [Array<String>] the x axis labels, evenly spread
+      def labels
+        count = [LABELS, @series.size].min
+        return [] if count.zero?
+
+        (0...count).map { label(@series[count == 1 ? 0 : ((@series.size - 1) * _1 / (count - 1).to_f).round][:date]) }
+      end
+
+      # @return [Array<Hash>] per day the hover column +left+/+width+ and the dot +top+, in percent, with the numbers
+      def points # rubocop:disable Metrics/AbcSize
+        slot = 100.0 / [@series.size, 1].max
+        @series.each_with_index.map do |day, index|
+          center = @series.size == 1 ? 50.0 : x(index) / WIDTH * 100
+          day.merge(left: (center - (slot / 2)).round(2), width: slot.round(2), center: center,
+                    top: (y(day[:count]) / HEIGHT * 100).round(2), label: label(day[:date]))
+        end
+      end
+
+      private
+
+      def x(index)
+        return WIDTH / 2.0 if @series.size == 1
+
+        (index * WIDTH / (@series.size - 1).to_f).round(1)
+      end
+
+      def y(value)
+        (HEIGHT - (value / @top.to_f * PLOT)).round(1)
+      end
+
+      # The smallest "nice" axis maximum (1, 2 or 5 times a power of ten per tick) at or above +max+.
+      def top(max)
+        return TICKS if max.zero?
+
+        raw = max / TICKS.to_f
+        magnitude = 10**Math.log10(raw).floor
+        step = [1, 2, 5, 10].map { _1 * magnitude }.find { _1 >= raw }
+        step * TICKS
+      end
+
+      def label(date)
+        "#{date[4, 2]}/#{date[6, 2]}"
+      end
+    end
+  end
+end