cosmonats 0.6.0 → 0.8.0

data/lib/cosmo/api/cron.rb

@@ -10,41 +10,42 @@ module Cosmo
     # Derives the schedule list entirely from NATS.
     # Whatever is deployed in NATS is exactly what appears in the UI.
     #
-    # Schedule templates live in the same job stream they target (e.g. +default+),
-    # stored at subjects matching +cosmo.cron.<stream>.>+. NATS 2.14 fires each
-    # template by publishing the body to +Nats-Schedule-Target+ as a regular
-    # JetStream message that accumulates alongside pending jobs.
+    # Every schedule template lives in the +scheduled+ stream, at a subject matching
+    # +cosmo.cron.<target stream>.>+. NATS fires a template by publishing its body to
+    # +Nats-Schedule-Target+, which has to be a subject the storing stream covers, so a firing
+    # lands back in +scheduled+ and the job processor's scheduler dispatches it to the stream
+    # that runs it. That confines message scheduling - and the +discard: old+ NATS demands
+    # wherever scheduling is enabled - to a single stream.
     class Cron
+      STREAM = ::Cosmo::Job::StreamFilter::SCHEDULED.to_s
+
       def self.instance
         @instance ||= new
       end
 
       # @return [Array<Hash>] every cron schedule currently deployed in NATS
       def all
-        Stream.jobs.flat_map { |s| schedules_from_stream(s.name) }
+        schedules
       rescue StandardError
         []
       end
 
       # Publish (or replace) a schedule message in NATS.
       # @return [Hash, nil] the persisted schedule as a hash, or nil on failure
+      # @raise [SchedulingDisabledError] when scheduling is turned off
       def upsert!(class_name: nil, stream: nil, schedule: nil, args: [], timezone: nil, name: nil)
+        raise SchedulingDisabledError unless Config.scheduled.enabled
+
         e = Entry.new(class_name: class_name, stream: stream, expression: schedule,
                       args: args, timezone: timezone, name: name)
-        headers = {
-          "Nats-Schedule" => e.expression,
-          "Nats-Schedule-Target" => e.target_subject
-        }
-        headers["Nats-Schedule-Time-Zone"] = e.timezone if e.timezone
-        client.publish(e.schedule_subject, e.job_payload, stream: e.stream, header: headers)
-        build_from_nats(e.stream, e.schedule_subject)
+        client.publish(e.schedule_subject, e.job_payload, stream: STREAM, header: e.schedule_headers)
+        build_from_nats(e.schedule_subject)
       end
 
       # Purge the schedule message from NATS (stops future firings).
       # @param subject [String]
       def delete!(subject)
-        stream_name = subject.to_s.split(".")[2]
-        client.purge(stream_name, subject)
+        client.purge(STREAM, subject)
       rescue NATS::JetStream::Error::NotFound, NATS::IO::Timeout
         nil
       end
@@ -52,13 +53,12 @@ module Cosmo
       # Dispatch the job immediately to the target stream, bypassing the timer.
       # @param schedule_subject [String] e.g. "cosmo.cron.default.report_job.daily"
       def run_now!(schedule_subject) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        stream_name = schedule_subject.to_s.split(".")[2]
-        msg = client.get_message(stream_name, subject: schedule_subject)
+        msg = client.get_message(STREAM, subject: schedule_subject)
         return unless msg
 
         headers = msg.headers || {}
         body = Utils::Json.parse(msg.data) || {}
-        target = headers["Nats-Schedule-Target"]
+        target = headers["X-Subject"]
         return unless target && body[:class]
 
         payload = Utils::Json.dump({
@@ -68,7 +68,7 @@ module Cosmo
                                      retry: body[:retry] || Job::Data.default_retry,
                                      dead: body[:dead].nil? ? Job::Data::DEFAULTS[:dead] : body[:dead]
                                    })
-        client.publish(target, payload, stream: stream_name)
+        client.publish(target, payload, stream: headers["X-Stream"])
       rescue NATS::JetStream::Error::NotFound
         nil
       end
@@ -79,14 +79,13 @@ module Cosmo
         @client ||= Client.instance
       end
 
-      def schedules_from_stream(stream_name)
-        filter = "#{Entry::SUBJECT_PREFIX}.#{stream_name}.>"
-        subjects = client.cron_subjects_in_stream(stream_name, filter)
-        subjects.filter_map { |subj| build_from_nats(stream_name, subj) }
+      def schedules
+        subjects = client.cron_subjects_in_stream(STREAM, "#{Entry::SUBJECT_PREFIX}.>")
+        subjects.filter_map { |subj| build_from_nats(subj) }
       end
 
-      def build_from_nats(stream_name, subject)
-        msg = client.get_message(stream_name, subject: subject)
+      def build_from_nats(subject)
+        msg = client.get_message(STREAM, subject: subject)
         return unless msg
 
         headers = msg.headers || {}
@@ -94,13 +93,13 @@ module Cosmo
 
         {
           class: body[:class],
-          stream: stream_name,
+          stream: headers["X-Stream"],
           schedule: headers["Nats-Schedule"],
           timezone: headers["Nats-Schedule-Time-Zone"],
           args: body[:args] || [],
           name: name_from_subject(subject),
           schedule_subject: subject,
-          target_subject: headers["Nats-Schedule-Target"],
+          dispatch_subject: headers["X-Subject"],
           registry_key: subject.split(".").drop(2).join("/")
         }
       rescue StandardError

data/lib/cosmo/api/job.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "time"
+
 module Cosmo
   module API
     class Job
@@ -38,9 +40,37 @@ module Cosmo
         @message.subject
       end
 
+      def error_class
+        headers&.dig("X-Error-Class")
+      end
+
+      def error_message
+        headers&.dig("X-Error-Message")
+      end
+
+      def error_backtrace
+        headers&.dig("X-Error-Backtrace")
+      end
+
+      # @return [Array<String>] the backtrace frames, which travel as a single header line
+      def error_backtrace_lines
+        error_backtrace.to_s.split(::Cosmo::Job::Failure::BACKTRACE_SEPARATOR)
+      end
+
+      def error?
+        !error_class.to_s.empty? || !error_message.to_s.empty?
+      end
+
       def timestamp
         headers&.dig("Nats-Time-Stamp")
       end
+
+      # @return [Time, nil] when the message was stored in its stream, parsed from +Nats-Time-Stamp+
+      def stored_at
+        Time.iso8601(timestamp) if timestamp
+      rescue ::ArgumentError
+        nil
+      end
     end
   end
 end

data/lib/cosmo/api/kv.rb

@@ -3,6 +3,8 @@
 module Cosmo
   module API
     class KV
+      LIMIT = 25
+
       attr_reader :kv
 
       def initialize(name, options = nil)
@@ -39,17 +41,19 @@ module Cosmo
         kv.delete(key)
       end
 
-      def keys(subject = nil, limit: 25)
-        results = []
-        watcher = kv.watch(subject || ">", ignore_deletes: true, meta_only: true)
-        watcher.each do |entry|
-          break unless entry
+      # Walks the bucket and collects live keys. Pass +limit: nil+ to walk the whole
+      # bucket -- a bounded +limit+ stops the watch early, so it can never be used
+      # to count (see #count).
+      def keys(subject = nil, limit: LIMIT, offset: 0)
+        walk(subject, limit:, offset:, meta_only: true).map(&:first)
+      end
 
-          results << entry.key
-          break if results.size >= limit
-        end
-        watcher.stop
-        results
+      # Same walk as #keys, but carries each value back with its key. The watch
+      # already has the values in hand, so callers that need them get one pass
+      # instead of a #keys walk followed by a #get per key.
+      # @return [Array<Array(String, String)>] +[key, value]+ pairs
+      def entries(subject = nil, limit: LIMIT, offset: 0)
+        walk(subject, limit:, offset:, meta_only: false)
       end
 
       # Writes a KV-Operation tombstone (same issue as #delete on ttl buckets).
@@ -70,7 +74,7 @@ module Cosmo
       end
 
       def count
-        keys.size
+        keys(limit: nil).size
       rescue NATS::KeyValue::NoKeysFoundError, NATS::JetStream::Error::NotFound
         0
       end
@@ -78,6 +82,22 @@ module Cosmo
 
       private
 
+      def walk(subject, limit:, offset:, meta_only:)
+        results = []
+        skipped = 0
+        watcher = kv.watch(subject || ">", ignore_deletes: true, meta_only:)
+        watcher.each do |entry|
+          break unless entry
+
+          next skipped += 1 if skipped < offset.to_i
+
+          results << [entry.key, entry.value]
+          break if limit && results.size >= limit
+        end
+        watcher.stop
+        results
+      end
+
       # CAS = Compare-And-Swap: publish +value+ with a per-message Nats-TTL,
       # but only if the subject's current last sequence matches +last_seq+
       # (sent as the Nats-Expected-Last-Subject-Sequence header). Raises

data/lib/cosmo/api/registry.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    # Base for a KV bucket of JSON entries that live only while their writer keeps refreshing them:
+    # a writer re-puts its entries faster than the bucket's TTL, and an entry whose writer is gone expires.
+    # Subclasses define +BUCKET+ and +TTL+.
+    class Registry
+      LIMIT = 25
+
+      def self.instance
+        @instance ||= new
+      end
+
+      def initialize
+        @kv = bucket
+      end
+
+      # Writes or refreshes an entry, recreating the bucket if it was deleted under a running writer.
+      #
+      # @param key [String, Integer]
+      # @param value [Hash]
+      # @return [void]
+      def put(key, value)
+        @kv.set(sanitize(key), Utils::Json.dump(value))
+      rescue NATS::JetStream::Error::NoStreamResponse
+        @kv = bucket
+        @kv.set(sanitize(key), Utils::Json.dump(value))
+      end
+
+      # Removes an entry, leaving no tombstone behind.
+      #
+      # @param key [String, Integer]
+      # @return [void]
+      def remove(key)
+        @kv.erase(sanitize(key))
+      end
+
+      # @param page [Integer, nil]
+      # @param limit [Integer]
+      # @return [Array<Hash>]
+      def list(page: nil, limit: LIMIT)
+        offset = ([page.to_i, 1].max - 1) * limit
+        @kv.entries(limit:, offset:).filter_map { Utils::Json.parse(_1.last) }
+      end
+
+      # @return [Array<Hash>] every live entry
+      def all
+        @kv.entries(limit: nil).filter_map { Utils::Json.parse(_1.last) }
+      end
+
+      # @return [Integer] the number of live entries
+      def size
+        @kv.size
+      end
+
+      private
+
+      def bucket
+        KV.new(self.class::BUCKET, { ttl: self.class::TTL, replicas: Config.replicas })
+      end
+
+      def sanitize(key)
+        key.to_s.gsub(/[^\w.=-]/, "_")
+      end
+    end
+  end
+end

data/lib/cosmo/api/stats/busy.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module Cosmo
+  module API
+    module Stats
+      # Jobs being executed right now, one entry per in-flight message, across all workers.
+      class Busy < Registry
+        TTL = 70
+        HEARTBEAT = 30
+        BUCKET = "_cosmobusy"
+
+        def initialize
+          super
+          @messages = {}
+        end
+
+        def with(message)
+          add(message)
+          yield
+        ensure
+          delete(message)
+        end
+
+        def add(message)
+          @thread ||= Thread.new { heartbeat_loop }
+          meta = message.metadata
+          seq = meta.sequence.stream
+          entry = { data: message.data, stream: meta.stream, worker: worker_id,
+                    started_at: Time.now.to_i, delivery: meta.num_delivered }
+          @messages[seq] = entry
+          put(seq, entry)
+        end
+
+        def delete(message)
+          seq = message.metadata.sequence.stream
+          @messages.delete(seq)
+          remove(seq)
+        end
+
+        def list(...)
+          super.map { _1.merge(data: Utils::Json.parse(_1[:data])) }
+        end
+
+        private
+
+        def heartbeat_loop
+          loop do
+            sleep(HEARTBEAT)
+            @messages.dup.each { |seq, entry| put(seq, entry) rescue StandardError }
+          rescue StandardError => e
+            Logger.debug "Busy heartbeat error: #{e.class} #{e.message}"
+          end
+        end
+
+        def worker_id
+          @worker_id ||= "#{Socket.gethostname}-#{::Process.pid}"
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/api/stats/metrics.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    module Stats
+      # Per-day metrics for every job class: runs, failures, and execution and wait time, kept in their own counters
+      # stream for +config.metrics.retention+. Workers {#record} executions in memory and the heartbeat {#flush}es them,
+      # so a job costs no NATS round trip; a crashed worker loses at most one heartbeat's worth.
+      class Metrics < Counter
+        STREAM_NAME = "_cosmometrics"
+        DESCRIPTION = "Cosmo metrics"
+        DAY = 86_400
+
+        def self.instance
+          @instance ||= new
+        end
+
+        # Each day's counters expire +config.metrics.retention+ after their last update.
+        #
+        # @return [Hash]
+        def self.stream_config
+          super.merge(max_age: Config.to_ns(Utils::Duration.parse(Config.metrics.retention)))
+        end
+
+        def initialize
+          super("jobs")
+          @lock = Mutex.new
+          @buffer = buffer
+        end
+
+        # Adds one execution to the buffer {#flush} writes.
+        #
+        # @param job_class [String]
+        # @param exec_ms [Numeric, nil] how long a successful execution took
+        # @param wait_ms [Numeric, nil] time from enqueue to the job's first delivery
+        # @param failed [Boolean]
+        # @return [void]
+        def record(job_class, exec_ms: nil, wait_ms: nil, failed: false)
+          key = [Time.now.utc.strftime("%Y%m%d"), job_class.gsub("::", "-")]
+          @lock.synchronize do
+            totals = @buffer[key]
+            failed ? totals[:failed] += 1 : totals[:count] += 1
+            totals[:exec_ms] += exec_ms if exec_ms
+            next unless wait_ms
+
+            totals[:waited] += 1
+            totals[:wait_ms] += wait_ms
+          end
+        end
+
+        # Writes the buffered executions to the counters and empties the buffer.
+        #
+        # @return [void]
+        def flush
+          pending = @lock.synchronize { @buffer.tap { @buffer = buffer } }
+          pending.each do |(day, job), totals|
+            totals.each { |field, value| increment("#{day}.#{job}.#{field}", by: value.round) if value.positive? }
+          end
+        end
+
+        # @param days [Integer] today and the days before it
+        # @return [Array<Hash>] per job class +:job+, +:count+, +:failed+, +:exec_ms+ and +:wait_ms+ (averages in
+        #   milliseconds, nil without data), busiest first
+        def summary(days: 1)
+          totals = Hash.new { |hash, job| hash[job] = Hash.new(0) }
+          read(days).each { |(_, job, field), value| totals[job][field] += value }
+          totals.map { |job, sums| row(sums).merge(job: job) }.sort_by { -_1[:count] }
+        end
+
+        # @param job_class [String]
+        # @param days [Integer]
+        # @return [Array<Hash>] one +{ date:, count:, failed:, exec_ms:, wait_ms: }+ per day, oldest first
+        def daily(job_class, days: 30)
+          sums = Hash.new { |hash, day| hash[day] = Hash.new(0) }
+          read(days).each { |(day, job, field), value| sums[day][field] += value if job == job_class }
+          dates(days).map { |day| row(sums[day]).merge(date: day) }
+        end
+
+        private
+
+        def buffer
+          Hash.new { |hash, key| hash[key] = Hash.new(0) }
+        end
+
+        def read(days)
+          filters = dates(days).map { "#{STREAM_NAME}.#{@namespace}.#{_1}.>" }
+          client.last_messages(STREAM_NAME, filters).map do |subject, data|
+            day, job, field = subject.split(".").last(3)
+            [[day, job.gsub("-", "::"), field.to_sym], Utils::Json.parse(data, default: {})[:val].to_i]
+          end
+        end
+
+        def dates(days)
+          now = Time.now.utc
+          (days - 1).downto(0).map { (now - (_1 * DAY)).strftime("%Y%m%d") }
+        end
+
+        def row(sums)
+          { count: sums[:count], failed: sums[:failed], exec_ms: average(sums[:exec_ms], sums[:count]),
+            wait_ms: average(sums[:wait_ms], sums[:waited]) }
+        end
+
+        def average(total, count)
+          total.fdiv(count).round(1) if count.positive?
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/api/stats/processes.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    module Stats
+      # Live worker processes, one entry per process, refreshed by Cosmo::Heartbeat.
+      class Processes < Registry
+        TTL = 60
+        BUCKET = "_cosmoprocesses"
+        KINDS = %w[jobs streams jobs+streams none].freeze
+
+        # Which processors a process is pulling for, judged by its non-empty subscriptions.
+        #
+        # @param process [Hash]
+        # @return [String] one of {KINDS}
+        def self.kind(process)
+          types = Hash(process[:subscriptions]).reject { |_, names| Array(names).empty? }.keys.map(&:to_s).sort
+          types.empty? ? "none" : types.join("+")
+        end
+
+        # Sorted by kind, host and pid before paging: every heartbeat rewrites an entry, so the bucket's own
+        # order shifts constantly and would move processes between pages, and each kind stays contiguous.
+        #
+        # @param page [Integer, nil]
+        # @param limit [Integer]
+        # @return [Array<Hash>]
+        def list(page: nil, limit: LIMIT)
+          offset = ([page.to_i, 1].max - 1) * limit
+          all.sort_by { [KINDS.index(self.class.kind(_1)) || KINDS.size, _1[:hostname].to_s, _1[:pid].to_i] }.slice(offset, limit).to_a
+        end
+
+        # @param identity [String] a unique process id, e.g. +hostname-pid+
+        # @param info [Hash] the process details shown in the Web UI
+        # @return [void]
+        def register(identity, info)
+          put(identity, info)
+        end
+
+        # @param identity [String]
+        # @return [void]
+        def unregister(identity)
+          remove(identity)
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/api/stats/totals.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    module Stats
+      # Processed and failed job totals across all workers, in the +_cosmototals+ counters. They never expire.
+      class Totals < Counter
+        STREAM_NAME = "_cosmototals"
+        DESCRIPTION = "Cosmo statistics"
+
+        def self.instance
+          @instance ||= new("jobs")
+        end
+
+        # @return [Integer]
+        def processed
+          get(:processed)
+        end
+
+        # @return [Integer]
+        def failed
+          get(:failed)
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/api/stats.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
-require "cosmo/api/counter"
-require "cosmo/api/busy"
+require "cosmo/api/stats/totals"
+require "cosmo/api/stats/busy"
+require "cosmo/api/stats/processes"
+require "cosmo/api/stats/metrics"
 
 module Cosmo
   module API
@@ -13,11 +15,11 @@ module Cosmo
       end
 
       def processed
-        Counter.instance.get(:processed)
+        Totals.instance.processed
       end
 
       def failed
-        Counter.instance.get(:failed)
+        Totals.instance.failed
       end
 
       def busy
@@ -33,11 +35,11 @@ module Cosmo
       end
 
       def scheduled
-        Stream.new("scheduled").size
+        Config.scheduled.enabled ? Stream.new("scheduled").size : 0
       end
 
       def dead
-        Stream.new("dead").size
+        Config.dead.enabled ? Stream.new("dead").size : 0
       end
     end
   end

data/lib/cosmo/api/stream.rb

@@ -36,7 +36,7 @@ module Cosmo
 
       def total
         all_msgs = info[:state].messages.to_i
-        cron_count = Client.instance.cron_subjects_in_stream(name, "#{Cron::Entry::SUBJECT_PREFIX}.#{name}.>").size
+        cron_count = Client.instance.cron_subjects_in_stream(name, "#{Cron::Entry::SUBJECT_PREFIX}.>").size
         [all_msgs - cron_count, 0].max
       rescue NATS::Error
         0

data/lib/cosmo/api.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
 
-require "cosmo/api/stream"
-require "cosmo/api/counter"
 require "cosmo/api/kv"
+require "cosmo/api/counter"
+require "cosmo/api/registry"
+require "cosmo/api/stream"
 require "cosmo/api/stats"
 require "cosmo/api/cron"
 require "cosmo/api/batch"

data/lib/cosmo/batch/counters.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "cosmo/api/counter"
+
+module Cosmo
+  class Batch
+    # Every batch's total, pending, and failed counters, in the +_cosmobatches+ stream. They are purged once the batch
+    # finishes, and expire +config.batches.expiry+ after their last update otherwise, along with the batch's KV state.
+    class Counters < API::Counter
+      STREAM_NAME = "_cosmobatches"
+      DESCRIPTION = "Cosmo batch progress"
+
+      # @return [Hash]
+      def self.stream_config
+        super.merge(max_age: Config.to_ns(Utils::Duration.parse(Config.batches.expiry)))
+      end
+    end
+  end
+end