cosmonats 0.6.0 → 0.8.0

data/lib/cosmo/batch.rb

@@ -3,6 +3,7 @@
 require "securerandom"
 require "cosmo/batch/callback"
 require "cosmo/batch/dispatcher"
+require "cosmo/batch/counters"
 
 module Cosmo
   # Groups jobs together and runs a callback once they've all finished.
@@ -39,13 +40,20 @@ module Cosmo
   # when everything succeeds. Always call #jobs at least once, even with nothing
   # in it - a batch that never gets a #jobs call never closes and never fires.
   #
+  # You can pass your own bid instead of letting one be generated - useful
+  # when something else (e.g. an ActiveRecord id) needs to match it:
+  #
+  #   Cosmo::Batch.new(bid: your_id)
+  #
+  # Same meaning as a plain #new either way - it's still a fresh batch.
+  #
   # == How this is actually stored in NATS
   #
   # Every batch has an id (bid, a random hex string). State lives
   # in two places, both keyed by that bid:
   #
-  # 1. Three atomic counters (via API::Counter, namespace "batch", backed by
-  #    the shared _cosmostats stream):
+  # 1. Three atomic counters (via Batch::Counters, namespace "batch", in the
+  #    _cosmobatches stream, which expires them with the KV state below):
   #
   #      <bid>.total    - how many jobs have ever joined this batch
   #      <bid>.pending  - how many of them are still running
@@ -76,8 +84,7 @@ module Cosmo
   class Batch
     extend Dispatcher
 
-    BUCKET = "cosmo_jobs_batches"
-    DEFAULT_EXPIRY = "3d"
+    BUCKET = "_cosmobatches"
     EVENTS = %i[success complete].freeze
 
     def self.current
@@ -93,17 +100,17 @@ module Cosmo
     end
 
     def self.counter
-      @counter ||= API::Counter.new("batch")
+      @counter ||= Counters.new("batch")
     end
 
     def self.kv
-      @kv ||= API::KV.new(BUCKET, ttl: Utils::Duration.parse(Config[:batch_expiry] || DEFAULT_EXPIRY))
+      @kv ||= API::KV.new(BUCKET, ttl: Utils::Duration.parse(Config.batches.expiry), replicas: Config.replicas)
     end
 
     attr_reader :bid, :parent_id
 
-    def initialize(parent: nil)
-      @bid = SecureRandom.hex(8)
+    def initialize(bid: nil, parent: nil)
+      @bid = bid || SecureRandom.hex(8)
       @parent_id = parent
       kv.set("#{@bid}.meta", Utils::Json.dump({ parent_id: @parent_id, created_at: Time.now.to_i }))
       link(@parent_id) if @parent_id

data/lib/cosmo/cli.rb

@@ -2,9 +2,10 @@
 
 require "yaml"
 require "optparse"
+require "fileutils"
 
 module Cosmo
-  # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/BlockLength
+  # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/BlockLength
   class CLI
     def self.run
       instance.run
@@ -14,15 +15,23 @@ module Cosmo
       @instance ||= new
     end
 
+    # @return [Array<String>, nil] the arguments the process was started with, before parsing consumed them
+    attr_reader :argv
+
     def run
+      @argv = ARGV.dup
       flags, command, options = parse
+      return run_init if flags[:init]
       return run_setup(flags) if flags[:setup]
 
       load_config(flags)
+      Engine.processors_for(command).each { _1.validate_options!(options) }
       puts self.class.banner
       boot_application
       require_path(flags[:require])
       Engine.run(command, options)
+    rescue Error => e
+      abort e.message
     end
 
     private
@@ -50,9 +59,19 @@ module Cosmo
         path = default_path if File.exist?(default_path)
       end
 
-      Config.load(path)
-      Config.set(:concurrency, flags[:concurrency]) if flags[:concurrency]
-      Config.set(:timeout, flags[:timeout]) if flags[:timeout]
+      overrides = { concurrency: flags[:concurrency], timeout: flags[:timeout] }.compact
+      overrides[:http] = { port: flags[:http_port] } if flags[:http_port]
+      Config.load(path, overrides:)
+    end
+
+    def run_init
+      path = File.expand_path(Config::DEFAULT_PATH)
+      raise Error, "#{Config::DEFAULT_PATH} already exists" if File.exist?(path)
+
+      FileUtils.mkdir_p(File.dirname(path))
+      FileUtils.cp(Config::DEFAULTS_FILE, path)
+      puts "Created #{Config::DEFAULT_PATH} with Cosmo's defaults"
+      exit(0)
     end
 
     def run_setup(flags)
@@ -69,9 +88,12 @@ module Cosmo
           first_line ? print("Stream is ready: #{name}") : print(", #{name}")
           first_line = false
         end
+        puts unless first_line
       end
 
-      puts
+      services = Services.setup!
+      puts "Service streams are ready: #{services.join(", ")}" if services.any?
+
       schedules = Config.dig(:setup, :cron)&.reduce(0) do |sum, (name, entry)|
         class_name = entry.delete(:class)
         API::Cron.instance.upsert!(**entry, name: name, class_name: class_name)
@@ -129,6 +151,10 @@ module Cosmo
           flags[:timeout] = arg
         end
 
+        o.on "-p", "--http-port INT", Integer, "Start HTTP server with /health endpoint on port" do |arg|
+          flags[:http_port] = arg
+        end
+
         o.on "-C", "--config PATH", "Path to config file" do |arg|
           flags[:config_file] = arg
         end
@@ -137,6 +163,10 @@ module Cosmo
           flags[:setup] = true
         end
 
+        o.on "--init", "Write #{Config::DEFAULT_PATH} with Cosmo's defaults, then exit" do
+          flags[:init] = true
+        end
+
         o.on_tail "-v", "--version", "Print version" do
           puts "Cosmo #{VERSION}"
           exit(0)
@@ -155,8 +185,16 @@ module Cosmo
         OptionParser.new do |o|
           o.banner = "Usage: cosmo jobs [options]"
 
-          o.on "--stream NAME", "Job's stream" do |arg|
-            options[:stream] = arg
+          o.on "--streams NAMES", "Only subscribe to these job streams (comma-separated), instead of all configured streams" do |arg|
+            options[:streams] = Array(options[:streams]) | arg.split(",")
+          end
+
+          o.on "--stream NAME", "Same as --streams, for a single job stream" do |arg|
+            options[:streams] = Array(options[:streams]) | [arg]
+          end
+
+          o.on "--[no-]scheduler", "Dispatch scheduled jobs (default: yes)" do |arg|
+            options[:scheduler] = arg
           end
 
           o.on "--subject NAME", "Job's subject" do |arg|
@@ -214,7 +252,7 @@ module Cosmo
       require path
     end
 
-    # rubocop:disable Layout/TrailingWhitespace,Lint/IneffectiveAccessModifier
+    # rubocop:disable-next Layout/TrailingWhitespace,Lint/IneffectiveAccessModifier
     def self.banner
       <<-TEXT
                     .#%+:                                                  
@@ -254,7 +292,5 @@ module Cosmo
                  :.                                                        
       TEXT
     end
-    # rubocop:enable Layout/TrailingWhitespace,Lint/IneffectiveAccessModifier
   end
-  # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/BlockLength
 end

data/lib/cosmo/client.rb

@@ -1,21 +1,49 @@
 # frozen_string_literal: true
 
+require "socket"
 require "nats/client"
 require "cosmo/utils/overrides"
 
 module Cosmo
   class Client
+    # Seconds to wait for a JetStream API reply. Override with COSMO_JS_TIMEOUT.
+    JS_TIMEOUT = 5
+    # Seconds to wait for the TCP connect+handshake. Override with COSMO_CONNECT_TIMEOUT.
+    CONNECT_TIMEOUT = 2
+
     def self.instance
       @instance ||= Client.new
     end
 
-    attr_reader :nc, :js
+    # Labels the connection in `nats server report connections` and /connz, so a
+    # misbehaving process can be found without cross-referencing host IPs. Derived from
+    # the running program, which separates a web process from a `cosmo` worker on the
+    # same host. Override with COSMO_CLIENT_NAME.
+    def self.default_name
+      ENV.fetch("COSMO_CLIENT_NAME") { "cosmo-#{program_name}-#{Socket.gethostname}-#{Process.pid}" }
+    end
+
+    def self.js_timeout
+      ENV.fetch("COSMO_JS_TIMEOUT", JS_TIMEOUT).to_i
+    end
+
+    def self.connect_timeout
+      ENV.fetch("COSMO_CONNECT_TIMEOUT", CONNECT_TIMEOUT).to_i
+    end
+
+    # $PROGRAM_NAME is "puma 7.2.1 (tcp://...)" under Puma and a path under the CLI.
+    def self.program_name
+      File.basename($PROGRAM_NAME.to_s.split.first.to_s, ".*").gsub(/[^\w.-]/, "")
+    end
+
+    attr_reader :nc, :js, :name
 
-    def initialize(nats_url: ENV.fetch("NATS_URL", "nats://localhost:4222"))
-      Logger.debug "Connecting to NATS server at #{nats_url}..."
-      @nc = NATS.connect(nats_url)
+    def initialize(nats_url: ENV.fetch("NATS_URL", "nats://localhost:4222"), name: self.class.default_name)
+      @name = name
+      Logger.debug "Connecting to NATS server at #{nats_url} as #{@name}..."
+      @nc = NATS.connect(nats_url, name: @name, connect_timeout: self.class.connect_timeout)
       Logger.debug "Connection established"
-      @js = @nc.jetstream
+      @js = @nc.jetstream(timeout: self.class.js_timeout)
     end
 
     def publish(subject, payload, **params)
@@ -116,6 +144,31 @@ module Cosmo
       js.get_msg(stream_name, **options)
     end
 
+    # The last message on every subject matching +filters+, in one batched direct get (NATS ≥ 2.11). Needs
+    # +allow_direct+ on the stream.
+    #
+    # @param stream_name [String]
+    # @param filters [Array<String>] subject filters, wildcards allowed
+    # @return [Hash{String => String}] message data by subject
+    def last_messages(stream_name, filters) # rubocop:disable Metrics/AbcSize
+      inbox = nc.new_inbox
+      sub = nc.subscribe(inbox)
+      nc.publish("$JS.API.DIRECT.GET.#{stream_name}", Utils::Json.dump({ multi_last: filters, batch: 10_000 }), inbox)
+      messages = {}
+      loop do
+        msg = sub.next_msg(timeout: self.class.js_timeout)
+        subject = msg.header&.dig("Nats-Subject")
+        break unless subject
+
+        messages[subject] = msg.data
+      end
+      messages
+    rescue NATS::Timeout
+      messages
+    ensure
+      sub&.unsubscribe
+    end
+
     def delete_message(name, seq)
       response = nc.request("$JS.API.STREAM.MSG.DELETE.#{name}", JSON.dump({ seq: seq }))
       Utils::Json.parse(response.data, symbolize_names: false)
@@ -170,7 +223,7 @@ module Cosmo
 
     # NOTE: KV manager in nats-pure hardcodes the fields it copies into StreamConfig,
     # so `allow_msg_ttl` is never forwarded via create_key_value. Send the raw stream-create API request instead.
-    def create_kv_with_msg_ttl(name, **options)
+    def create_kv_with_msg_ttl(name, replicas: 1, **options)
       payload = Utils::Json.dump({
         name: "KV_#{name}",
         subjects: ["$KV.#{name}.>"],
@@ -178,7 +231,8 @@ module Cosmo
         allow_direct: true,
         allow_msg_ttl: true,
         allow_rollup_hdrs: true,
-        max_msgs_per_subject: 1
+        max_msgs_per_subject: 1,
+        num_replicas: replicas
       }.merge(options))
       resp = nc.request("$JS.API.STREAM.CREATE.KV_#{name}", payload)
       result = Utils::Json.parse(resp.data, symbolize_names: false)

data/lib/cosmo/config/cosmo.yml

@@ -0,0 +1,58 @@
+# Cosmo configuration. Every value below is what Cosmo uses when you leave it out,
+# so you can delete anything you don't want to change. Listing your own job streams under setup.jobs
+# replaces the `default` stream and its consumer below. Scheduled jobs, dead jobs, and stats are
+# managed by Cosmo itself: tune them in Cosmo.configure instead.
+
+timeout: 25    # Shutdown timeout in seconds
+concurrency: 1 # Number of worker threads
+max_retries: 3 # Retries for a job class that sets none
+
+#http:          # Serve GET /health and /ping
+#  port: 9090
+#  host: 0.0.0.0
+
+stream_config: &stream_config
+  storage: file         # file or memory
+  retention: workqueue  # workqueue deletes acked messages; limits keeps them until max_age/max_msgs
+  duplicate_window: 120 # seconds a Nats-Msg-Id is remembered, so a resent job is stored once
+  discard: new          # what to drop when the stream is full: new rejects publishes, old evicts the oldest
+  allow_direct: true    # required by the web UI
+  subjects:
+    - jobs.%{name}.>    # %{name} is the stream name
+
+consumer_config: &consumer_config
+  ack_policy: explicit    # every message is acked on its own
+  max_deliver: 30         # deliveries before a message is given up on; keep it above every job class's retry
+  max_ack_pending: 20     # unacked messages before the server stops delivering more
+  ack_wait: 60            # seconds before an unacked message is redelivered
+  subject: jobs.%{name}.> # %{name} is the stream name
+
+setup:
+  jobs:
+    default:
+      <<: *stream_config
+      description: Default jobs
+#    critical:
+#      <<: *stream_config
+#      description: Critical jobs
+#  streams:               # Streams for Cosmo::Stream processors
+#    events:
+#      storage: file
+#      retention: limits
+#      max_age: 86400
+#      subjects:
+#        - events.>
+#  cron:                  # Recurring jobs, synced by `cosmo --setup`
+#    daily_report:
+#      class: ReportJob
+#      schedule: "@daily"
+#      stream: default
+
+consumers:
+  jobs:
+    default:
+      <<: *consumer_config
+      priority: 15         # relative polling weight among job streams
+#    critical:
+#      <<: *consumer_config
+#      priority: 50

data/lib/cosmo/config/settings.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class Config < ::Hash
+    # +config.scheduled+ in Cosmo.configure. Turned off, Cosmo neither creates the scheduled stream nor runs the
+    # scheduler, and delayed jobs (+perform_in+/+perform_at+) and crons raise {SchedulingDisabledError}.
+    Scheduled = Struct.new(:enabled, keyword_init: true)
+
+    # +config.dead+ in Cosmo.configure: the dead-letter stream's retention (+max_age+ in seconds or a duration such as
+    # "7d"). Turned off, a job that gives up is dropped instead of parked.
+    Dead = Struct.new(:enabled, :max_age, :max_msgs, :max_bytes, keyword_init: true)
+
+    # +config.batches+ in Cosmo.configure: how long a batch's tracking data lives (seconds or a duration such as "3d").
+    Batches = Struct.new(:expiry, keyword_init: true)
+
+    # +config.metrics+ in Cosmo.configure: per-day job metrics behind the web UI's Metrics tab, kept for +retention+
+    # (seconds or a duration such as "30d"). Turned off, nothing is recorded and the tab is hidden.
+    Metrics = Struct.new(:enabled, :retention, keyword_init: true)
+  end
+end

data/lib/cosmo/config.rb

@@ -2,31 +2,68 @@
 
 require "yaml"
 require "forwardable"
+require "cosmo/config/settings"
 
 module Cosmo
   class Config < ::Hash
     NANO = 1_000_000_000
     DEFAULT_PATH = "config/cosmo.yml"
+    DEFAULTS_FILE = File.expand_path(DEFAULT_PATH, __dir__)
+    SERVICE_STREAMS = %i[scheduled dead].freeze
 
     class << self
       extend Forwardable
 
-      delegate %i[[] fetch dig to_h set load] => :instance
+      delegate %i[[] fetch dig load server_middleware client_middleware error_handlers replicas scheduled dead batches metrics] => :instance
     end
 
     def self.to_ns(seconds)
       (seconds.to_f * NANO).to_i
     end
 
+    def self.read(path)
+      Utils::Hash.symbolize_keys!(YAML.load_file(path, aliases: true) || {})
+    end
+
     def self.parse_file(path)
-      YAML.load_file(path, aliases: true).tap { normalize!(_1) }
+      read(path).tap { normalize!(_1) }
+    end
+
+    # The built-in {DEFAULTS_FILE} with +user+ deep-merged over it. Job streams are the one list that is not merged:
+    # a user who lists any in +setup.jobs+ gets exactly those, and the built-in +default+ stream and consumer go away.
+    # Without +setup.jobs+, +consumers.jobs.default+ still tunes the built-in consumer.
+    #
+    # @param user [Hash] a parsed config file, not yet normalized
+    # @return [Hash] the normalized effective config
+    # @raise [ConfigError] when +user+ configures something Cosmo.configure owns
+    def self.build(user)
+      validate!(user)
+      defaults = read(DEFAULTS_FILE)
+      if user.dig(:setup, :jobs)
+        defaults[:setup].delete(:jobs)
+        defaults[:consumers].delete(:jobs)
+      end
+      Utils::Hash.deep_merge(defaults, user).tap { normalize!(_1) }
+    end
+
+    def self.validate!(config)
+      %i[setup consumers].each do |section|
+        jobs = config.dig(section, :jobs)
+        name = SERVICE_STREAMS.find { jobs.is_a?(::Hash) && jobs.key?(_1) }
+        next unless name
+
+        raise ConfigError, "`#{section}.jobs.#{name}` is a Cosmo service stream: remove it from cosmo.yml " \
+                           "and tune it with Cosmo.configure { |config| config.#{name} }"
+      end
+      return unless config.key?(:batch_expiry)
+
+      raise ConfigError, "`batch_expiry` moved out of cosmo.yml: Cosmo.configure { |config| config.batches.expiry = ... }"
     end
 
     def self.normalize!(config) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       Utils::Hash.symbolize_keys!(config)
 
       config[:timeout] = Utils::Duration.parse(config[:timeout]) if config[:timeout]
-      config[:batch_expiry] = Utils::Duration.parse(config[:batch_expiry]) if config[:batch_expiry]
 
       config[:consumers]&.each_key do |name|
         config[:consumers][name].each do |stream_name, c|
@@ -46,13 +83,6 @@ module Cosmo
           c[:max_age] = to_ns(Utils::Duration.parse(c[:max_age])) if c[:max_age]
           c[:duplicate_window] = to_ns(Utils::Duration.parse(c[:duplicate_window])) if c[:duplicate_window]
           c[:subjects] = c[:subjects].map { |s| format(s, name: name) } if c[:subjects]
-
-          next unless type == :jobs # Every jobs stream supports NATS 2.14 message scheduling.
-
-          c[:allow_msg_schedules] = true
-          cron_subject = "#{API::Cron::Entry::SUBJECT_PREFIX}.#{name}.>"
-          c[:subjects] = Array(c[:subjects])
-          c[:subjects] << cron_subject unless c[:subjects].include?(cron_subject)
         end
       end
     end
@@ -80,14 +110,113 @@ module Cosmo
       @internal ||= {}
     end
 
-    def set(...)
-      Utils::Hash.set(self, ...)
+    # Replaces the contents with {.build} of the file at +path+ (or of no file), with +overrides+ merged over it, so
+    # command-line flags win over the file.
+    #
+    # @param path [String, nil]
+    # @param overrides [Hash, nil] e.g. +{ concurrency: 5 }+
+    def load(path = nil, overrides: nil)
+      user = path ? self.class.read(path) : {}
+      replace(self.class.build(Utils::Hash.deep_merge(user, Hash(overrides))))
+    end
+
+    attr_writer :replicas
+
+    # @return [Integer] replicas for every service stream and bucket, e.g. 3 on a NATS cluster
+    def replicas
+      @replicas || 1
+    end
+
+    # @return [Scheduled]
+    def scheduled
+      @scheduled ||= Scheduled.new(enabled: true)
     end
 
-    def load(path = nil)
-      return unless path
+    # @return [Dead]
+    def dead
+      @dead ||= Dead.new(enabled: true, max_age: 7 * 86_400, max_msgs: 10_000, max_bytes: -1)
+    end
+
+    # @return [Batches]
+    def batches
+      @batches ||= Batches.new(expiry: 3 * 86_400)
+    end
+
+    # @return [Metrics]
+    def metrics
+      @metrics ||= Metrics.new(enabled: true, retention: "30d")
+    end
+
+    # Callables given every error Cosmo rescues: failed jobs and stream batches, fetch and scheduler errors, rejected
+    # messages. Each is called with +(error, context)+, +context+ being a Hash with a +:source+ (+:job+, +:stream+,
+    # +:fetch+, +:scheduler+, +:reject+, +:retry_in+, +:limit+) and what is known there, e.g. a job's payload.
+    #
+    #   Cosmo.configure do |config|
+    #     config.error_handlers << ->(error, context) { Honeybadger.notify(error, context: context) }
+    #   end
+    #
+    # @return [Array<#call>]
+    def error_handlers
+      @error_handlers ||= []
+    end
+
+    # @return [::Logger] the logger Cosmo writes to, see {Logger.instance}
+    def logger
+      Logger.instance
+    end
+
+    # Replaces Cosmo's stdout logger, e.g. with +Rails.logger+. Give it +Cosmo::Logger::SimpleFormatter.new+ to keep
+    # Cosmo's +jid+/+elapsed+ context in each line.
+    #
+    # @param logger [::Logger]
+    def logger=(logger)
+      Logger.instance = logger
+    end
+
+    # Ignored when +COSMO_LOG_LEVEL+ is set, which always wins.
+    #
+    # @param level [Symbol, String, Integer] +:trace+, +:debug+, +:info+, +:warn+, +:error+, or +:fatal+
+    def log_level=(level)
+      Logger.level = level
+    end
+
+    # The chain every job execution runs through, starting with the built-in {Middleware::Limit}, {Middleware::Busy},
+    # {Middleware::Totals}, and {Middleware::Metrics}. It lives outside the loaded YAML, so {#load} keeps it. Register middleware
+    # at boot, before workers start:
+    #
+    #   Cosmo.configure do |config|
+    #     config.server_middleware do |chain|
+    #       chain.add MyMiddleware
+    #     end
+    #   end
+    #
+    # @yieldparam chain [Middleware::Chain]
+    # @return [Middleware::Chain]
+    def server_middleware
+      @server_middleware ||= Middleware::Chain.new do |chain|
+        chain.add Middleware::Limit
+        chain.add Middleware::Busy
+        chain.add Middleware::Totals
+        chain.add Middleware::Metrics
+      end
+      yield @server_middleware if block_given?
+      @server_middleware
+    end
 
-      replace(self.class.parse_file(path))
+    # The chain every job enqueue runs through, empty by default. A middleware is called with
+    # +(job_class_name, payload, stream)+ around the publish: it may change +payload+ (the Hash that gets published),
+    # or not yield to stop the publish, so +perform_async+ returns +nil+. +stream+ is the stream the job runs on.
+    #
+    #   Cosmo.configure do |config|
+    #     config.client_middleware { |chain| chain.add RequestIdMiddleware }
+    #   end
+    #
+    # @yieldparam chain [Middleware::Chain]
+    # @return [Middleware::Chain]
+    def client_middleware
+      @client_middleware ||= Middleware::Chain.new
+      yield @client_middleware if block_given?
+      @client_middleware
     end
   end
 end