cosmonats 0.7.0 → 0.8.0

data/lib/cosmo/processor.rb

@@ -86,9 +86,10 @@ module Cosmo
           end
           all_paused = false
 
-          _, skip_t = consumer_state[stream_name]
+          key = backoff_key(config)
+          _, skip_t = consumer_state[key]
           if skip_t && Time.now < skip_t
-            Logger.trace "stream #{stream_name} is empty, backing off"
+            Logger.trace "#{key} is empty, backing off"
             next
           end
           all_empty = false
@@ -96,7 +97,7 @@ module Cosmo
           begin
             @pool.post do
               # Re-check, after possibly being blocked on post to thread pool
-              _, skip_t = consumer_state[stream_name]
+              _, skip_t = consumer_state[key]
               next if skip_t && Time.now < skip_t
 
               timeout = fetch_timeout(config)
@@ -105,11 +106,11 @@ module Cosmo
               messages = fetch(subscription, batch_size: config[:batch_size], timeout:)
               Logger.trace "fetched (#{messages&.size.to_i}) messages"
               if messages&.any?
-                consumer_state.delete(stream_name)
+                consumer_state.delete(key)
                 track_busy { process(messages, processor) }
               else
                 max_backoff = Utils::Duration.parse(ENV.fetch("COSMO_STREAM_EMPTY_BACKOFF_MAX", STREAM_EMPTY_BACKOFF_MAX))
-                consumer_state.compute(stream_name) do |(count, wake_at)|
+                consumer_state.compute(key) do |(count, wake_at)|
                   next [count, wake_at] if wake_at && wake_at > started_at
 
                   count = count.to_i + 1
@@ -179,6 +180,7 @@ module Cosmo
     rescue StandardError => e
       Logger.error "Snap! Error just happened"
       Logger.error "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
+      Cosmo.handle_error(e, { source: :fetch })
 
       backoff = ENV.fetch("COSMO_STREAMS_FETCH_BACKOFF", 5).to_f
       sleep([timeout, backoff].max) # backoff before retry
@@ -196,5 +198,9 @@ module Cosmo
     def consumer_state
       @consumer_state ||= Concurrent::Map.new
     end
+
+    def backoff_key(config)
+      config[:stream].to_s
+    end
   end
 end

data/lib/cosmo/publisher.rb

@@ -7,7 +7,7 @@ module Cosmo
     class << self
       extend Forwardable
 
-      delegate %i[publish publish_job publish_batch] => :instance
+      delegate %i[publish] => :instance
     end
 
     def self.instance
@@ -22,17 +22,5 @@ module Cosmo
       payload = (serializer || Stream::Serializer).serialize(data)
       @client.publish(subject, payload, **options)
     end
-
-    def publish_job(data)
-      subject, payload, params = data.to_args
-      @client.publish(subject, payload, **params)
-      data.jid
-    rescue NATS::JetStream::Error::NoStreamResponse
-      raise StreamNotFoundError, params[:stream].to_s
-    end
-
-    def publish_batch(subject, batch, **options)
-      batch.each { publish(subject, _1, **options) }
-    end
   end
 end

data/lib/cosmo/railtie.rb

@@ -12,10 +12,10 @@ module Cosmo
       require "cosmo/active_job"
     end
 
-    # Autoload config/cosmo.yml when it exists and no config has been loaded yet.
+    # Loads config/cosmo.yml, or the built-in defaults without one, unless a config has been loaded already.
     initializer "cosmo.load_config", after: "cosmo.active_job_adapter" do |app|
       config_path = app.root.join("config", "cosmo.yml")
-      Config.load(config_path.to_s) if config_path.exist? && Config.instance.none?
+      Config.load(config_path.exist? ? config_path.to_s : nil) if Config.instance.none?
     end
   end
 end

data/lib/cosmo/services.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Cosmo
+  # The streams Cosmo itself depends on. They are built from the settings in Cosmo.configure, never from cosmo.yml,
+  # and `cosmo --setup` creates them next to the user's streams.
+  module Services
+    SCHEDULED = Job::StreamFilter::SCHEDULED.to_s
+    DEAD = "dead"
+    METADATA = { "_cosmo.type" => "jobs" }.freeze
+    RETENTION_CHANGE = 10_052
+
+    module_function
+
+    # Holds delayed jobs and every cron template. It alone enables NATS message scheduling, since a schedule can only
+    # target a subject its own stream covers, and NATS refuses +discard: new+ wherever scheduling is enabled.
+    #
+    # @return [Hash, nil] nil when scheduling is turned off
+    def scheduled_stream
+      return unless Config.scheduled.enabled
+
+      { storage: "file", retention: "workqueue", discard: "old", allow_direct: true, allow_msg_schedules: true,
+        duplicate_window: Config.to_ns(120), subjects: ["jobs.#{SCHEDULED}.>", "#{API::Cron::Entry::SUBJECT_PREFIX}.>"],
+        num_replicas: Config.replicas, description: "Scheduled jobs and cron schedules", metadata: METADATA }
+    end
+
+    # +max_deliver+ stays above 1: the scheduler naks a message that is not yet due, and one whose dispatch failed,
+    # and both need another delivery.
+    #
+    # @return [Hash, nil] nil when scheduling is turned off
+    def scheduled_consumer
+      return unless Config.scheduled.enabled
+
+      { ack_policy: "explicit", max_deliver: 5, max_ack_pending: 100, ack_wait: 10, subject: "jobs.#{SCHEDULED}.>" }
+    end
+
+    # @return [Hash, nil] nil when dead-lettering is turned off
+    def dead_stream
+      dead = Config.dead
+      return unless dead.enabled
+
+      { storage: "file", retention: "workqueue", discard: "old", allow_direct: true, duplicate_window: Config.to_ns(120),
+        max_age: Config.to_ns(Utils::Duration.parse(dead.max_age)), max_msgs: dead.max_msgs, max_bytes: dead.max_bytes,
+        subjects: ["jobs.#{DEAD}.>"], num_replicas: Config.replicas, description: "Dead jobs", metadata: METADATA }
+    end
+
+    # Creates or updates every enabled service stream, plus the internal totals and batch counters and, while enabled,
+    # the job metrics.
+    #
+    # @return [Array<String>] the names of the service streams set up, without the internal ones
+    def setup!
+      streams = { SCHEDULED => scheduled_stream, DEAD => dead_stream }.compact
+      streams.each { |name, config| setup_stream(name, config) }
+      API::Stats::Totals.setup!
+      Batch::Counters.setup!
+      API::Stats::Metrics.setup! if Config.metrics.enabled
+      streams.keys
+    end
+
+    # @raise [Error] when an existing stream has another retention policy, which NATS can't change in place
+    def setup_stream(name, config)
+      Client.instance.setup_stream(name, config)
+    rescue NATS::JetStream::Error::APIError => e
+      raise unless e.err_code == RETENTION_CHANGE
+
+      raise Error, "The existing `#{name}` stream has another retention policy, which NATS can't change in place: " \
+                   "delete it (`nats stream rm #{name}`, dropping its messages) and rerun `cosmo --setup`"
+    end
+  end
+end

data/lib/cosmo/stream/processor.rb

@@ -38,8 +38,10 @@ module Cosmo
         processor.process(messages)
         Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "done" }
       rescue StandardError => e
-        Logger.debug e
+        Logger.error e
         Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
+        Cosmo.handle_error(e, { source: :stream, stream: metadata&.stream, processor: processor.class.name,
+                                size: messages&.size })
       rescue Exception # rubocop:disable Lint/RescueException
         Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
         raise
@@ -70,6 +72,10 @@ module Cosmo
         config.dig(:consumer, :subjects)
       end
 
+      def backoff_key(config)
+        config[:consumer_name] || config[:class].name
+      end
+
       def fetch_timeout(config)
         timeout = config[:fetch_timeout].to_f
         if timeout <= 0

data/lib/cosmo/utils/hash.rb

@@ -36,14 +36,13 @@ module Cosmo
         end
       end
 
-      # deep set
-      def set(hash, *keys, value)
-        last_key = keys.pop
-        target = keys.reduce(hash) do |base, key|
-          base[key] ||= {}
-          base[key]
-        end
-        target[last_key] = value
+      # Nested hashes merge recursively, +other+ winning; any other value in +other+ replaces the one in +base+.
+      #
+      # @param base [::Hash]
+      # @param other [::Hash]
+      # @return [::Hash] a new hash
+      def deep_merge(base, other)
+        base.merge(other) { |_, old, new| old.is_a?(::Hash) && new.is_a?(::Hash) ? deep_merge(old, new) : new }
       end
 
       # deep dup

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/lib/cosmo/web/assets/app.css

@@ -439,6 +439,15 @@ td {
 tr:last-child td { border-bottom: none; }
 tr:hover         { background: var(--color-table-bg-alt); }
 
+tr.group-row td,
+tr.group-row:hover { background: var(--color-table-bg-alt); }
+tr.group-row td {
+  color: var(--color-text-light);
+  font-size: var(--font-size-small);
+  font-weight: 600;
+  padding: var(--space) var(--space-2x);
+}
+
 /* ── Subjects ──────────────────────────────────────────────────────────── */
 .subjects { display: flex; flex-wrap: wrap; gap: var(--space-1-2); }
 
@@ -539,6 +548,15 @@ details summary {
 details summary code { display: inline; user-select: text; }
 details[open] summary { margin-bottom: var(--space); }
 details code { display: block; max-width: 400px; white-space: pre-wrap; }
+details summary:has(> code.truncate) { white-space: nowrap; }
+details summary code.truncate {
+  display: inline-block;
+  max-width: 32ch;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  vertical-align: middle;
+  white-space: nowrap;
+}
 
 /* ── HTMX ──────────────────────────────────────────────────────────────── */
 .htmx-indicator { opacity: 0; transition: opacity 200ms; }
@@ -563,6 +581,46 @@ details code { display: block; max-width: 400px; white-space: pre-wrap; }
 }
 
 /* ── Responsive ────────────────────────────────────────────────────────── */
+
+.metrics-chart {
+  background: var(--color-elevated);
+  border-radius: var(--space-1-2);
+  box-shadow: 0 1px 3px var(--color-shadow);
+  padding: var(--space-2x) var(--space-3x);
+  margin-bottom: var(--space-3x);
+}
+.metrics-chart svg { width: 100%; height: auto; display: block; }
+.metrics-chart-header {
+  display: flex;
+  justify-content: space-between;
+  flex-wrap: wrap;
+  gap: var(--space);
+  margin-bottom: var(--space);
+}
+.metrics-legend { display: flex; gap: var(--space-2x); font-size: var(--font-size-small); color: var(--color-text-light); }
+.metrics-legend span::before {
+  content: "";
+  display: inline-block;
+  width: 10px;
+  height: 10px;
+  margin-right: 6px;
+  border-radius: 2px;
+}
+.metrics-legend-runs::before { background: var(--color-primary); opacity: 0.45; }
+.metrics-legend-exec::before { background: var(--color-warning); }
+.chart-grid  { stroke: var(--color-border); stroke-width: 1; }
+.chart-axis  { fill: var(--color-text-light); font-size: 11px; font-family: var(--font-sans); }
+.chart-bar   { fill: var(--color-primary); opacity: 0.45; }
+.chart-bar:hover { opacity: 0.7; }
+.chart-line  { fill: none; stroke: var(--color-warning); stroke-width: 2; }
+.chart-point { fill: var(--color-warning); stroke: var(--color-elevated); stroke-width: 2; }
+.metrics-hint { margin: 0 0 var(--space); font-size: var(--font-size-small); color: var(--color-text-light); }
+.metrics-table tr.selectable { cursor: pointer; }
+.metrics-table tr.selectable:hover { background: var(--color-selected); }
+.metrics-table tr.selected { background: var(--color-selected); }
+.metrics-table tr.selected td:first-child { box-shadow: inset 3px 0 0 var(--color-primary); }
+.metrics-table tr.selected code { color: var(--color-primary); }
+
 @media (max-width: 768px) {
   .nav            { flex-wrap: wrap; }
   .nav-list       { order: 3; width: 100%; margin-top: var(--space); }

data/lib/cosmo/web/chart.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class Web
+    # Geometry for the Metrics tab's SVG chart: a bar per day for the number of runs on the left axis, and a line for
+    # the average execution time on the right one.
+    class Chart
+      WIDTH = 760
+      HEIGHT = 240
+      TOP = 16
+      BOTTOM = 212
+      LEFT = 52
+      RIGHT = 700
+      TICKS = 4
+      LABELS = 10
+
+      # @param series [Array<Hash>] +{ date:, count:, exec_ms: }+ per day, oldest first
+      def initialize(series)
+        @series = series
+        @count_top = top(series.map { _1[:count] }.max.to_i)
+        @exec_top = top(series.filter_map { _1[:exec_ms] }.max.to_f)
+      end
+
+      # @return [Boolean]
+      def empty?
+        @series.all? { _1[:count].zero? }
+      end
+
+      # @return [Array<Hash>] +{ x:, y:, width:, height:, title: }+
+      def bars # rubocop:disable Metrics/AbcSize
+        @series.each_with_index.map do |day, index|
+          height = scale(day[:count], @count_top)
+          { x: (slot_x(index) + (slot * 0.15)).round(1), y: (BOTTOM - height).round(1), width: (slot * 0.7).round(1),
+            height: height.round(1), title: "#{label(day[:date])}: #{day[:count]} run(s)" }
+        end
+      end
+
+      # @return [Array<Hash>] +{ x:, y:, title: }+ for the days with an average
+      def points
+        @series.each_with_index.filter_map do |day, index|
+          next unless day[:exec_ms]
+
+          { x: (slot_x(index) + (slot / 2)).round(1), y: (BOTTOM - scale(day[:exec_ms], @exec_top)).round(1),
+            title: "#{label(day[:date])}: #{day[:exec_ms]} ms on average" }
+        end
+      end
+
+      # @return [String] the +points+ attribute of the execution time polyline
+      def line
+        points.map { "#{_1[:x]},#{_1[:y]}" }.join(" ")
+      end
+
+      # @return [Array<Hash>] +{ y:, count:, exec_ms: }+ per horizontal grid line, bottom up
+      def ticks
+        (0..TICKS).map do |step|
+          { y: (BOTTOM - ((BOTTOM - TOP) * step / TICKS.to_f)).round(1),
+            count: (@count_top * step / TICKS.to_f).round, exec_ms: (@exec_top * step / TICKS.to_f).round }
+        end
+      end
+
+      # @return [Array<Hash>] +{ x:, text: }+, thinned out to at most LABELS
+      def labels
+        every = (@series.size / LABELS.to_f).ceil.clamp(1, nil)
+        @series.each_with_index.filter_map do |day, index|
+          { x: (slot_x(index) + (slot / 2)).round(1), text: label(day[:date]) } if (index % every).zero?
+        end
+      end
+
+      private
+
+      def slot
+        (RIGHT - LEFT) / [@series.size, 1].max.to_f
+      end
+
+      def slot_x(index)
+        LEFT + (slot * index)
+      end
+
+      def scale(value, top)
+        top.zero? ? 0 : (BOTTOM - TOP) * value / top.to_f
+      end
+
+      # The smallest "nice" axis maximum (1, 2 or 5 times a power of ten per tick) at or above +max+.
+      def top(max)
+        return TICKS if max.zero?
+
+        raw = max / TICKS.to_f
+        magnitude = 10**Math.log10(raw).floor
+        step = [1, 2, 5, 10].map { _1 * magnitude }.find { _1 >= raw }
+        step * TICKS
+      end
+
+      def label(date)
+        "#{date[4, 2]}/#{date[6, 2]}"
+      end
+    end
+  end
+end

data/lib/cosmo/web/controllers/crons.rb

@@ -7,17 +7,23 @@ module Cosmo
     module Controllers
       class Crons < Application
         def index
+          return not_found unless Config.scheduled.enabled
+
           content_for :title, "Crons"
           ok render("crons/index", layout: true)
         end
 
         def _table
+          return not_found unless Config.scheduled.enabled
+
           ok render("crons/_table", { schedules: cron.all })
         end
 
         # Dispatch the job immediately, bypassing the schedule timer.
         # Expects params["subject"] = the schedule subject stored in NATS.
         def run_now
+          return not_found unless Config.scheduled.enabled
+
           subject = Rack::Utils.unescape(params["subject"].to_s)
           cron.run_now!(subject)
           ok
@@ -25,6 +31,8 @@ module Cosmo
 
         # Purge the schedule from NATS so it stops firing.
         def delete
+          return not_found unless Config.scheduled.enabled
+
           subject = Rack::Utils.unescape(params["subject"].to_s)
           cron.delete!(subject)
           _table

data/lib/cosmo/web/controllers/jobs.rb

@@ -8,7 +8,8 @@ module Cosmo
       class Jobs < Application
         def index
           content_for :title, "Jobs"
-          ok render("jobs/index", layout: true)
+          processes_query = params.slice("page", "limit").transform_values(&:to_i).select { _2.positive? }
+          ok render("jobs/index", { processes_query: }, layout: true)
         end
 
         def busy
@@ -27,6 +28,8 @@ module Cosmo
         end
 
         def scheduled
+          return not_found unless Config.scheduled.enabled
+
           return _scheduled if hx_request?
 
           content_for :title, "Scheduled Jobs"
@@ -34,6 +37,8 @@ module Cosmo
         end
 
         def dead
+          return not_found unless Config.dead.enabled
+
           return _dead if hx_request?
 
           content_for :title, "Dead Jobs"
@@ -41,6 +46,8 @@ module Cosmo
         end
 
         def retry
+          return not_found unless Config.dead.enabled
+
           seq = path.split("/").last.to_i
           stream = API::Stream.new("dead")
           stream.retry(seq)
@@ -48,6 +55,8 @@ module Cosmo
         end
 
         def delete
+          return not_found unless Config.dead.enabled
+
           seq = path.split("/").last.to_i
           stream = API::Stream.new("dead")
           stream.delete(seq)
@@ -62,12 +71,16 @@ module Cosmo
         end
 
         def _scheduled
+          return not_found unless Config.scheduled.enabled
+
           stream = API::Stream.new("scheduled")
           jobs = stream.messages(page: params["page"], limit: params["limit"])
           ok render("jobs/_scheduled", { jobs: jobs, total: stream.total })
         end
 
         def _dead
+          return not_found unless Config.dead.enabled
+
           stream = API::Stream.new("dead")
           jobs = stream.messages(page: params["page"], limit: params["limit"])
           ok render("jobs/_dead", { jobs: jobs, total: stream.total })
@@ -86,7 +99,8 @@ module Cosmo
           total = processes.size
           page, limit, total_pages = paginate(total, API::Stats::Processes::LIMIT)
           polling = params["poll"].to_s != "0"
-          ok render("jobs/_processes", { processes: processes.list(page:, limit:), total:, page:, limit:, total_pages:, polling: })
+          groups = processes.list(page:, limit:).chunk { API::Stats::Processes.kind(_1) }.to_a
+          ok render("jobs/_processes", { groups:, total:, page:, limit:, total_pages:, polling: })
         end
 
         def _enqueued # rubocop:disable Metrics/AbcSize

data/lib/cosmo/web/controllers/metrics.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "cosmo/web/controllers/application"
+require "cosmo/web/chart"
+
+module Cosmo
+  class Web
+    module Controllers
+      class Metrics < Application
+        RANGES = { 1 => "Today", 7 => "7 days", 30 => "30 days" }.freeze
+
+        def index
+          return not_found unless Config.metrics.enabled
+
+          content_for :title, "Metrics"
+          ok render("metrics/index", { days:, ranges: RANGES }, layout: true)
+        end
+
+        # The chart and the table together, so the charted job stays highlighted across refreshes.
+        def _panel
+          return not_found unless Config.metrics.enabled
+
+          rows = metrics.summary(days:)
+          job = charted(rows)
+          chart = Chart.new(metrics.daily(job, days:)) if job && days > 1
+          ok render("metrics/_panel", { rows:, job:, days:, chart: })
+        end
+
+        private
+
+        # The requested job while it has rows in the period, the busiest one otherwise.
+        def charted(rows)
+          job = params["job"].to_s
+          rows.any? { _1[:job] == job } ? job : rows.first&.dig(:job)
+        end
+
+        def days
+          value = params["days"].to_i
+          RANGES.key?(value) ? value : 7
+        end
+
+        def metrics
+          API::Stats::Metrics.instance
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/web/helpers/application.rb

@@ -42,6 +42,16 @@ module Cosmo
           value.to_s
         end
 
+        # @param millis [Numeric, nil]
+        # @return [String] e.g. "340 ms", "1.25 s", "2m 3s", or "—" without a value
+        def duration(millis)
+          return "—" unless millis
+          return "#{millis.round} ms" if millis < 1_000
+          return "#{(millis / 1_000.0).round(2)} s" if millis < 60_000
+
+          "#{(millis / 60_000).floor}m #{((millis % 60_000) / 1_000).round}s"
+        end
+
         def elapsed(value)
           elapsed = Time.now.to_i - value.to_i
 

data/lib/cosmo/web/views/batches/_table.erb

@@ -8,7 +8,9 @@
           <th>Batch</th>
           <th>Parent</th>
           <th>Status</th>
+          <th>Progress</th>
           <th>Total</th>
+          <th>Pending</th>
           <th>Succeeded</th>
           <th>Failed</th>
           <th>Created</th>
@@ -17,6 +19,7 @@
       <tbody>
         <% @batches.each do |b| -%>
           <% stats = b.stats -%>
+          <% total = stats[:total].to_i -%>
           <tr>
             <td><code><%= h(b.bid) %></code></td>
             <td>
@@ -35,7 +38,15 @@
                 <span class="badge badge-success">Success</span>
               <% end -%>
             </td>
+            <td>
+              <% if total.positive? -%>
+                <%= ((total - stats[:pending].to_i) * 100 / total).clamp(0, 100) %>%
+              <% else -%>
+                <span class="text-muted">&mdash;</span>
+              <% end -%>
+            </td>
             <td><%= format_numbers(stats[:total]) %></td>
+            <td><%= format_numbers(stats[:pending]) %></td>
             <td><%= format_numbers(stats[:succeeded]) %></td>
             <td><%= format_numbers(stats[:failed]) %></td>
             <td><%= format_timestamp(b.created_at) %></td>