cosmonats 0.7.0 → 0.9.0

data/lib/cosmo/api/stats/busy.rb

@@ -9,7 +9,7 @@ module Cosmo
       class Busy < Registry
         TTL = 70
         HEARTBEAT = 30
-        BUCKET = "cosmo_jobs_busy"
+        BUCKET = "_cosmobusy"
 
         def initialize
           super
@@ -43,6 +43,11 @@ module Cosmo
           super.map { _1.merge(data: Utils::Json.parse(_1[:data])) }
         end
 
+        # @return [Hash{String => Integer}] the number of jobs running per stream, for the streams running any
+        def by_stream
+          all.group_by { _1[:stream] }.transform_values(&:size)
+        end
+
         private
 
         def heartbeat_loop

data/lib/cosmo/api/stats/metrics.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    module Stats
+      # Per-day metrics for every job class: runs, failures, and execution and wait time, kept in their own counters
+      # stream for +config.metrics.retention+. Workers {#record} executions in memory and the heartbeat {#flush}es them,
+      # so a job costs no NATS round trip; a crashed worker loses at most one heartbeat's worth.
+      class Metrics < Counter
+        STREAM_NAME = "_cosmometrics"
+        DESCRIPTION = "Cosmo metrics"
+        DAY = 86_400
+
+        def self.instance
+          @instance ||= new
+        end
+
+        # Each day's counters expire +config.metrics.retention+ after their last update.
+        #
+        # @return [Hash]
+        def self.stream_config
+          super.merge(max_age: Config.to_ns(Utils::Duration.parse(Config.metrics.retention)))
+        end
+
+        def initialize
+          super("jobs")
+          @lock = Mutex.new
+          @buffer = buffer
+        end
+
+        # Adds one execution to the buffer {#flush} writes.
+        #
+        # @param job_class [String]
+        # @param exec_ms [Numeric, nil] how long a successful execution took
+        # @param wait_ms [Numeric, nil] time from enqueue to the job's first delivery
+        # @param failed [Boolean]
+        # @return [void]
+        def record(job_class, exec_ms: nil, wait_ms: nil, failed: false)
+          key = [Time.now.utc.strftime("%Y%m%d"), job_class.gsub("::", "-")]
+          @lock.synchronize do
+            totals = @buffer[key]
+            failed ? totals[:failed] += 1 : totals[:count] += 1
+            totals[:exec_ms] += exec_ms if exec_ms
+            next unless wait_ms
+
+            totals[:waited] += 1
+            totals[:wait_ms] += wait_ms
+          end
+        end
+
+        # Writes the buffered executions to the counters and empties the buffer.
+        #
+        # @return [void]
+        def flush
+          pending = @lock.synchronize { @buffer.tap { @buffer = buffer } }
+          pending.each do |(day, job), totals|
+            totals.each { |field, value| increment("#{day}.#{job}.#{field}", by: value.round) if value.positive? }
+          end
+        end
+
+        # @param days [Integer] today and the days before it
+        # @return [Array<Hash>] per job class +:job+, +:count+, +:failed+, +:exec_ms+ and +:wait_ms+ (averages in
+        #   milliseconds, nil without data), busiest first
+        def summary(days: 1)
+          totals = Hash.new { |hash, job| hash[job] = Hash.new(0) }
+          read(days).each { |(_, job, field), value| totals[job][field] += value }
+          totals.map { |job, sums| row(sums).merge(job: job) }.sort_by { -_1[:count] }
+        end
+
+        # @param job_class [String, nil] nil sums every job class
+        # @param days [Integer]
+        # @return [Array<Hash>] one +{ date:, count:, failed:, exec_ms:, wait_ms: }+ per day, oldest first
+        def daily(job_class = nil, days: 30) # rubocop:disable Metrics/AbcSize
+          sums = Hash.new { |hash, day| hash[day] = Hash.new(0) }
+          read(days).each { |(day, job, field), value| sums[day][field] += value if job_class.nil? || job == job_class }
+          dates(days).map { |day| row(sums[day]).merge(date: day) }
+        end
+
+        private
+
+        def buffer
+          Hash.new { |hash, key| hash[key] = Hash.new(0) }
+        end
+
+        def read(days)
+          filters = dates(days).map { "#{STREAM_NAME}.#{@namespace}.#{_1}.>" }
+          client.last_messages(STREAM_NAME, filters).map do |subject, data|
+            day, job, field = subject.split(".").last(3)
+            [[day, job.gsub("-", "::"), field.to_sym], Utils::Json.parse(data, default: {})[:val].to_i]
+          end
+        end
+
+        def dates(days)
+          now = Time.now.utc
+          (days - 1).downto(0).map { (now - (_1 * DAY)).strftime("%Y%m%d") }
+        end
+
+        def row(sums)
+          { count: sums[:count], failed: sums[:failed], exec_ms: average(sums[:exec_ms], sums[:count]),
+            wait_ms: average(sums[:wait_ms], sums[:waited]) }
+        end
+
+        def average(total, count)
+          total.fdiv(count).round(1) if count.positive?
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/api/stats/processes.rb

@@ -6,17 +6,27 @@ module Cosmo
       # Live worker processes, one entry per process, refreshed by Cosmo::Heartbeat.
       class Processes < Registry
         TTL = 60
-        BUCKET = "cosmo_processes"
+        BUCKET = "_cosmoprocesses"
+        KINDS = %w[jobs streams jobs+streams none].freeze
 
-        # Sorted by host and pid before paging: every heartbeat rewrites an entry, so the bucket's own
-        # order shifts constantly and would move processes between pages.
+        # Which processors a process is pulling for, judged by its non-empty subscriptions.
+        #
+        # @param process [Hash]
+        # @return [String] one of {KINDS}
+        def self.kind(process)
+          types = Hash(process[:subscriptions]).reject { |_, names| Array(names).empty? }.keys.map(&:to_s).sort
+          types.empty? ? "none" : types.join("+")
+        end
+
+        # Sorted by kind, host and pid before paging: every heartbeat rewrites an entry, so the bucket's own
+        # order shifts constantly and would move processes between pages, and each kind stays contiguous.
         #
         # @param page [Integer, nil]
         # @param limit [Integer]
         # @return [Array<Hash>]
         def list(page: nil, limit: LIMIT)
           offset = ([page.to_i, 1].max - 1) * limit
-          all.sort_by { [_1[:hostname].to_s, _1[:pid].to_i] }.slice(offset, limit).to_a
+          all.sort_by { [KINDS.index(self.class.kind(_1)) || KINDS.size, _1[:hostname].to_s, _1[:pid].to_i] }.slice(offset, limit).to_a
         end
 
         # @param identity [String] a unique process id, e.g. +hostname-pid+

data/lib/cosmo/api/stats/totals.rb

@@ -3,23 +3,15 @@
 module Cosmo
   module API
     module Stats
-      # Processed and failed job totals across all workers.
+      # Processed and failed job totals across all workers, in the +_cosmototals+ counters. They never expire.
       class Totals < Counter
+        STREAM_NAME = "_cosmototals"
+        DESCRIPTION = "Cosmo statistics"
+
         def self.instance
           @instance ||= new("jobs")
         end
 
-        # Counts the block's outcome: +true+ is processed, +false+ or a raise is failed.
-        #
-        # @return [void]
-        def with
-          result = yield
-          increment(:processed) if result == true
-          increment(:failed) if result == false
-        rescue Exception # rubocop:disable Lint/RescueException
-          increment(:failed)
-        end
-
         # @return [Integer]
         def processed
           get(:processed)

data/lib/cosmo/api/stats.rb

@@ -3,6 +3,7 @@
 require "cosmo/api/stats/totals"
 require "cosmo/api/stats/busy"
 require "cosmo/api/stats/processes"
+require "cosmo/api/stats/metrics"
 
 module Cosmo
   module API
@@ -34,11 +35,26 @@ module Cosmo
       end
 
       def scheduled
-        Stream.new("scheduled").size
+        Config.scheduled.enabled ? Stream.new("scheduled").size : 0
       end
 
       def dead
-        Stream.new("dead").size
+        Config.dead.enabled ? Stream.new("dead").size : 0
+      end
+
+      # @return [Array<Hash>] a {.queue} row per job stream
+      def queues
+        busy = Busy.instance.by_stream
+        Stream.jobs.map { queue(_1, busy) }
+      end
+
+      # @param stream [Stream] a job stream
+      # @param busy [Hash{String => Integer}] running jobs per stream, as {Busy#by_stream} counts them
+      # @return [Hash] the stream's +:name+, waiting messages as +:size+, running jobs as +:busy+, redeliveries as
+      #   +:retries+, and whether it is +:paused+
+      def queue(stream, busy = Busy.instance.by_stream)
+        { name: stream.name, size: stream.size, busy: busy.fetch(stream.name, 0), retries: stream.retries,
+          paused: stream.paused? }
       end
     end
   end

data/lib/cosmo/api/stream.rb

@@ -44,9 +44,14 @@ module Cosmo
       alias size total
 
       def retries
-        client.list_consumers(name).sum { _1["num_redelivered"].to_i }
+        consumers.sum { _1["num_redelivered"].to_i }
+      end
+
+      # @return [Array<Hash>] the stream's consumers as JetStream reports them, with their config and delivery state
+      def consumers
+        client.list_consumers(name)
       rescue NATS::Error
-        0
+        []
       end
 
       def each

data/lib/cosmo/batch/counters.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "cosmo/api/counter"
+
+module Cosmo
+  class Batch
+    # Every batch's total, pending, and failed counters, in the +_cosmobatches+ stream. They are purged once the batch
+    # finishes, and expire +config.batches.expiry+ after their last update otherwise, along with the batch's KV state.
+    class Counters < API::Counter
+      STREAM_NAME = "_cosmobatches"
+      DESCRIPTION = "Cosmo batch progress"
+
+      # @return [Hash]
+      def self.stream_config
+        super.merge(max_age: Config.to_ns(Utils::Duration.parse(Config.batches.expiry)))
+      end
+    end
+  end
+end

data/lib/cosmo/batch.rb

@@ -3,6 +3,7 @@
 require "securerandom"
 require "cosmo/batch/callback"
 require "cosmo/batch/dispatcher"
+require "cosmo/batch/counters"
 
 module Cosmo
   # Groups jobs together and runs a callback once they've all finished.
@@ -51,8 +52,8 @@ module Cosmo
   # Every batch has an id (bid, a random hex string). State lives
   # in two places, both keyed by that bid:
   #
-  # 1. Three atomic counters (via API::Counter, namespace "batch", backed by
-  #    the shared _cosmostats stream):
+  # 1. Three atomic counters (via Batch::Counters, namespace "batch", in the
+  #    _cosmobatches stream, which expires them with the KV state below):
   #
   #      <bid>.total    - how many jobs have ever joined this batch
   #      <bid>.pending  - how many of them are still running
@@ -83,8 +84,7 @@ module Cosmo
   class Batch
     extend Dispatcher
 
-    BUCKET = "cosmo_jobs_batches"
-    DEFAULT_EXPIRY = "3d"
+    BUCKET = "_cosmobatches"
     EVENTS = %i[success complete].freeze
 
     def self.current
@@ -100,11 +100,11 @@ module Cosmo
     end
 
     def self.counter
-      @counter ||= API::Counter.new("batch")
+      @counter ||= Counters.new("batch")
     end
 
     def self.kv
-      @kv ||= API::KV.new(BUCKET, ttl: Utils::Duration.parse(Config[:batch_expiry] || DEFAULT_EXPIRY))
+      @kv ||= API::KV.new(BUCKET, ttl: Utils::Duration.parse(Config.batches.expiry), replicas: Config.replicas)
     end
 
     attr_reader :bid, :parent_id

data/lib/cosmo/cli.rb

@@ -2,6 +2,7 @@
 
 require "yaml"
 require "optparse"
+require "fileutils"
 
 module Cosmo
   # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/BlockLength
@@ -20,6 +21,7 @@ module Cosmo
     def run
       @argv = ARGV.dup
       flags, command, options = parse
+      return run_init if flags[:init]
       return run_setup(flags) if flags[:setup]
 
       load_config(flags)
@@ -57,10 +59,19 @@ module Cosmo
         path = default_path if File.exist?(default_path)
       end
 
-      Config.load(path)
-      Config.set(:concurrency, flags[:concurrency]) if flags[:concurrency]
-      Config.set(:timeout, flags[:timeout]) if flags[:timeout]
-      Config.set(:http, :port, flags[:http_port]) if flags[:http_port]
+      overrides = { concurrency: flags[:concurrency], timeout: flags[:timeout] }.compact
+      overrides[:http] = { port: flags[:http_port] } if flags[:http_port]
+      Config.load(path, overrides:)
+    end
+
+    def run_init
+      path = File.expand_path(Config::DEFAULT_PATH)
+      raise Error, "#{Config::DEFAULT_PATH} already exists" if File.exist?(path)
+
+      FileUtils.mkdir_p(File.dirname(path))
+      FileUtils.cp(Config::DEFAULTS_FILE, path)
+      puts "Created #{Config::DEFAULT_PATH} with Cosmo's defaults"
+      exit(0)
     end
 
     def run_setup(flags)
@@ -80,7 +91,8 @@ module Cosmo
         puts unless first_line
       end
 
-      API::Counter.setup!
+      services = Services.setup!
+      puts "Service streams are ready: #{services.join(", ")}" if services.any?
 
       schedules = Config.dig(:setup, :cron)&.reduce(0) do |sum, (name, entry)|
         class_name = entry.delete(:class)
@@ -151,6 +163,10 @@ module Cosmo
           flags[:setup] = true
         end
 
+        o.on "--init", "Write #{Config::DEFAULT_PATH} with Cosmo's defaults, then exit" do
+          flags[:init] = true
+        end
+
         o.on_tail "-v", "--version", "Print version" do
           puts "Cosmo #{VERSION}"
           exit(0)

data/lib/cosmo/client.rb

@@ -2,7 +2,6 @@
 
 require "socket"
 require "nats/client"
-require "cosmo/utils/overrides"
 
 module Cosmo
   class Client
@@ -10,6 +9,9 @@ module Cosmo
     JS_TIMEOUT = 5
     # Seconds to wait for the TCP connect+handshake. Override with COSMO_CONNECT_TIMEOUT.
     CONNECT_TIMEOUT = 2
+    # Reconnect attempts before the connection closes for good; -1 never gives up, so a process outlives a NATS
+    # outage. nats-pure lets NATS_MAX_RECONNECT_ATTEMPTS override it.
+    MAX_RECONNECT_ATTEMPTS = -1
 
     def self.instance
       @instance ||= Client.new
@@ -41,7 +43,8 @@ module Cosmo
     def initialize(nats_url: ENV.fetch("NATS_URL", "nats://localhost:4222"), name: self.class.default_name)
       @name = name
       Logger.debug "Connecting to NATS server at #{nats_url} as #{@name}..."
-      @nc = NATS.connect(nats_url, name: @name, connect_timeout: self.class.connect_timeout)
+      @nc = NATS.connect(nats_url, name: @name, connect_timeout: self.class.connect_timeout,
+                                   max_reconnect_attempts: MAX_RECONNECT_ATTEMPTS)
       Logger.debug "Connection established"
       @js = @nc.jetstream(timeout: self.class.js_timeout)
     end
@@ -144,6 +147,31 @@ module Cosmo
       js.get_msg(stream_name, **options)
     end
 
+    # The last message on every subject matching +filters+, in one batched direct get (NATS ≥ 2.11). Needs
+    # +allow_direct+ on the stream.
+    #
+    # @param stream_name [String]
+    # @param filters [Array<String>] subject filters, wildcards allowed
+    # @return [Hash{String => String}] message data by subject
+    def last_messages(stream_name, filters) # rubocop:disable Metrics/AbcSize
+      inbox = nc.new_inbox
+      sub = nc.subscribe(inbox)
+      nc.publish("$JS.API.DIRECT.GET.#{stream_name}", Utils::Json.dump({ multi_last: filters, batch: 10_000 }), inbox)
+      messages = {}
+      loop do
+        msg = sub.next_msg(timeout: self.class.js_timeout)
+        subject = msg.header&.dig(NATS::JetStream::Header::SUBJECT)
+        break unless subject
+
+        messages[subject] = msg.data
+      end
+      messages
+    rescue NATS::Timeout
+      messages
+    ensure
+      sub&.unsubscribe
+    end
+
     def delete_message(name, seq)
       response = nc.request("$JS.API.STREAM.MSG.DELETE.#{name}", JSON.dump({ seq: seq }))
       Utils::Json.parse(response.data, symbolize_names: false)
@@ -170,8 +198,9 @@ module Cosmo
 
     private
 
-    # NOTE: nats-pure's #pull_subscribe has no path to a true ephemeral pull consumer.
-    # Its rescue branch unconditionally sets `config[:durable_name] = durable` (jetstream.rb).
+    # NOTE: nats-pure's #pull_subscribe can't create a true ephemeral pull consumer: its rescue branch
+    # unconditionally sets `config[:durable_name] = durable` (jetstream.rb). So the consumer is created
+    # here, and #pull_subscribe only binds to it by name.
     def ephemeral_subscribe(subject, config) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       config = config.dup
       stream = config.delete(:stream) or raise ArgumentError, "stream required for ephemeral consumers"
@@ -186,19 +215,12 @@ module Cosmo
       end
 
       info = js.add_consumer(stream, consumer_config)
-
-      sub = nc.subscribe(nc.new_inbox)
-      sub.extend(NATS::JetStream.const_get(:PullSubscription))
-      sub.jsi = NATS::JetStream.const_get(:JS)::Sub.new(
-        js: js, stream: stream, consumer: info.name,
-        nms: "#{js.prefix}.CONSUMER.MSG.NEXT.#{stream}.#{info.name}"
-      )
-      sub
+      js.pull_subscribe(subject, nil, stream: stream, name: info.name)
     end
 
     # NOTE: KV manager in nats-pure hardcodes the fields it copies into StreamConfig,
     # so `allow_msg_ttl` is never forwarded via create_key_value. Send the raw stream-create API request instead.
-    def create_kv_with_msg_ttl(name, **options)
+    def create_kv_with_msg_ttl(name, replicas: 1, **options)
       payload = Utils::Json.dump({
         name: "KV_#{name}",
         subjects: ["$KV.#{name}.>"],
@@ -206,7 +228,8 @@ module Cosmo
         allow_direct: true,
         allow_msg_ttl: true,
         allow_rollup_hdrs: true,
-        max_msgs_per_subject: 1
+        max_msgs_per_subject: 1,
+        num_replicas: replicas
       }.merge(options))
       resp = nc.request("$JS.API.STREAM.CREATE.KV_#{name}", payload)
       result = Utils::Json.parse(resp.data, symbolize_names: false)

data/lib/cosmo/config/cosmo.yml

@@ -0,0 +1,58 @@
+# Cosmo configuration. Every value below is what Cosmo uses when you leave it out,
+# so you can delete anything you don't want to change. Listing your own job streams under setup.jobs
+# replaces the `default` stream and its consumer below. Scheduled jobs, dead jobs, and stats are
+# managed by Cosmo itself: tune them in Cosmo.configure instead.
+
+timeout: 25    # Shutdown timeout in seconds
+concurrency: 1 # Number of worker threads
+max_retries: 3 # Retries for a job class that sets none
+
+#http:          # Serve GET /health and /ping
+#  port: 9090
+#  host: 0.0.0.0
+
+stream_config: &stream_config
+  storage: file         # file or memory
+  retention: workqueue  # workqueue deletes acked messages; limits keeps them until max_age/max_msgs
+  duplicate_window: 120 # seconds a Nats-Msg-Id is remembered, so a resent job is stored once
+  discard: new          # what to drop when the stream is full: new rejects publishes, old evicts the oldest
+  allow_direct: true    # required by the web UI
+  subjects:
+    - jobs.%{name}.>    # %{name} is the stream name
+
+consumer_config: &consumer_config
+  ack_policy: explicit    # every message is acked on its own
+  max_deliver: 30         # deliveries before a message is given up on; keep it above every job class's retry
+  max_ack_pending: 20     # unacked messages before the server stops delivering more
+  ack_wait: 60            # seconds before an unacked message is redelivered
+  subject: jobs.%{name}.> # %{name} is the stream name
+
+setup:
+  jobs:
+    default:
+      <<: *stream_config
+      description: Default jobs
+#    critical:
+#      <<: *stream_config
+#      description: Critical jobs
+#  streams:               # Streams for Cosmo::Stream processors
+#    events:
+#      storage: file
+#      retention: limits
+#      max_age: 86400
+#      subjects:
+#        - events.>
+#  cron:                  # Recurring jobs, synced by `cosmo --setup`
+#    daily_report:
+#      class: ReportJob
+#      schedule: "@daily"
+#      stream: default
+
+consumers:
+  jobs:
+    default:
+      <<: *consumer_config
+      priority: 15         # relative polling weight among job streams
+#    critical:
+#      <<: *consumer_config
+#      priority: 50

data/lib/cosmo/config/settings.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class Config < ::Hash
+    # +config.scheduled+ in Cosmo.configure. Turned off, Cosmo neither creates the scheduled stream nor runs the
+    # scheduler, and delayed jobs (+perform_in+/+perform_at+) and crons raise {SchedulingDisabledError}.
+    Scheduled = Struct.new(:enabled, keyword_init: true)
+
+    # +config.dead+ in Cosmo.configure: the dead-letter stream's retention (+max_age+ in seconds or a duration such as
+    # "7d"). Turned off, a job that gives up is dropped instead of parked.
+    Dead = Struct.new(:enabled, :max_age, :max_msgs, :max_bytes, keyword_init: true)
+
+    # +config.batches+ in Cosmo.configure: how long a batch's tracking data lives (seconds or a duration such as "3d").
+    Batches = Struct.new(:expiry, keyword_init: true)
+
+    # +config.metrics+ in Cosmo.configure: per-day job metrics behind the web UI's Metrics tab, kept for +retention+
+    # (seconds or a duration such as "30d"). Turned off, nothing is recorded and the tab is hidden.
+    Metrics = Struct.new(:enabled, :retention, keyword_init: true)
+  end
+end