cosmonats 0.2.0 → 0.4.0

data/lib/cosmo/processor.rb

@@ -2,15 +2,23 @@
 
 module Cosmo
   class Processor
+    STREAM_PAUSED_RECHECK_TTL = 5.0 # Seconds a stream's paused state is cached before re-checking (override via COSMO_STREAM_PAUSED_RECHECK_TTL)
+    STREAMS_PAUSED_IDLE_SLEEP = 1.0 # Seconds to sleep when every stream is paused, preventing a tight CPU spin (override via COSMO_STREAMS_PAUSED_IDLE_SLEEP)
+    STREAM_EMPTY_BACKOFF_MAX = 5.0  # Max seconds to sleep between empty fetches (override via COSMO_STREAM_EMPTY_BACKOFF_MAX)
+
     def self.run(...)
       new(...).tap(&:run)
     end
 
+    attr_reader :consumers
+
     def initialize(pool, running, options)
       @pool = pool
       @running = running
       @options = options
+      @threads = []
       @consumers = []
+      @cache = Utils::TTLCache.new
     end
 
     def run
@@ -21,9 +29,95 @@ module Cosmo
       run_loop
     end
 
+    def stop(timeout = Config[:timeout])
+      @running.make_false
+      @pool.shutdown
+      @consumers.each { |(s, _)| s.unsubscribe rescue nil }
+      @pool.wait_for_termination(timeout)
+      @threads.compact.each { _1.join(timeout) || _1.kill }
+      @consumers.clear
+      @threads.clear
+    end
+
     private
 
     def run_loop
+      @threads << Thread.new { work_loop }
+      @threads << Thread.new { schedule_loop } if scheduler?
+    end
+
+    def work_loop # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      shutdown = false
+
+      while running?
+        break if shutdown
+
+        all_empty = true  # every stream is empty
+        all_paused = true # every stream is paused
+        consumers.each do |(subscription, config, processor)| # rubocop:disable Metrics/BlockLength
+          break unless running?
+
+          stream_name = config[:stream].to_s
+          ttl = ENV.fetch("COSMO_STREAM_PAUSED_RECHECK_TTL", STREAM_PAUSED_RECHECK_TTL).to_f
+          if @cache.fetch(stream_name, ttl:) { API::Stream.new(stream_name).paused? }
+            Logger.debug "stream #{stream_name} is paused, skipping fetch"
+            next
+          end
+          all_paused = false
+
+          _, skip_t = consumer_state[stream_name]
+          if skip_t && Time.now < skip_t
+            Logger.debug "stream #{stream_name} is empty, backing off"
+            next
+          end
+          all_empty = false
+
+          begin
+            @pool.post do
+              # Re-check, after possibly being blocked on post to thread pool
+              _, skip_t = consumer_state[stream_name]
+              next if skip_t && Time.now < skip_t
+
+              timeout = fetch_timeout(config)
+              Logger.debug "fetching #{fetch_subjects(config).inspect}, timeout=#{timeout}"
+              messages = lock(stream_name) { fetch(subscription, batch_size: config[:batch_size], timeout:) }
+              Logger.debug "fetched (#{messages&.size.to_i}) messages"
+              if messages&.any?
+                consumer_state.delete(stream_name)
+                process(messages, processor)
+              else
+                max_backoff = ENV.fetch("COSMO_STREAM_EMPTY_BACKOFF_MAX", STREAM_EMPTY_BACKOFF_MAX).to_f
+                consumer_state.compute(stream_name) do |current|
+                  count = (current&.first || 0) + 1
+                  backoff = [timeout * (2**(count - 1)), max_backoff].min
+                  [count, Time.now + backoff]
+                end
+              end
+            end
+          rescue Concurrent::RejectedExecutionError
+            shutdown = true
+            break # pool doesn't accept new jobs, we are shutting down
+          end
+        end
+
+        break unless running?
+
+        if all_paused
+          period = ENV.fetch("COSMO_STREAMS_PAUSED_IDLE_SLEEP", STREAMS_PAUSED_IDLE_SLEEP).to_f
+          Logger.debug "all streams paused, sleep=#{period}"
+          sleep(period)
+        elsif all_empty
+          next_wake = consumer_state.values.filter_map { |_, t| t }.min
+          next unless next_wake # entry was deleted concurrently (messages arrived), re-loop immediately
+
+          remaining = [next_wake - Time.now, 0.01].max
+          Logger.debug "all streams empty, sleep=#{remaining}"
+          sleep(remaining)
+        end
+      end
+    end
+
+    def schedule_loop
       raise NotImplementedError
     end
 
@@ -39,6 +133,10 @@ module Cosmo
       @running.true?
     end
 
+    def scheduler?
+      false
+    end
+
     def fetch(subscription, batch_size:, timeout:)
       subscription.fetch(batch_size, timeout:)
     rescue NATS::Timeout
@@ -49,6 +147,7 @@ module Cosmo
 
       backoff = ENV.fetch("COSMO_STREAMS_FETCH_BACKOFF", 5).to_f
       sleep([timeout, backoff].max) # backoff before retry
+      nil
     end
 
     def client
@@ -58,5 +157,14 @@ module Cosmo
     def stopwatch
       Utils::Stopwatch.new
     end
+
+    def lock(stream_name, &)
+      @locks ||= Hash.new { |h, k| h[k] = Mutex.new }
+      @locks[stream_name].synchronize(&)
+    end
+
+    def consumer_state
+      @consumer_state ||= Concurrent::Map.new
+    end
   end
 end

data/lib/cosmo/railtie.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Cosmo
+  # Rails Railtie — loaded automatically when cosmonats is required inside a
+  # Rails application. Ensures the ActiveJob adapter constant is registered
+  # before Rails tries to resolve it and autoloads +config/cosmo.yml+ when
+  # the file is present.
+  class Railtie < ::Rails::Railtie
+    # Make Cosmo::ActiveJobAdapter::Adapter available under the conventional
+    # ActiveJob namespace so :cosmonats resolves without any extra requires.
+    initializer "cosmo.active_job_adapter", before: :run_prepare_callbacks do
+      require "cosmo/active_job"
+    end
+
+    # Autoload config/cosmo.yml when it exists and no config has been loaded yet.
+    initializer "cosmo.load_config", after: "cosmo.active_job_adapter" do |app|
+      config_path = app.root.join("config", "cosmo.yml")
+      Config.load(config_path.to_s) if config_path.exist? && Config.instance.none?
+    end
+  end
+end

data/lib/cosmo/stream/processor.rb

@@ -3,51 +3,21 @@
 module Cosmo
   module Stream
     class Processor < ::Cosmo::Processor
-      def initialize(pool, running, options)
-        super
-        @configs = []
-      end
-
       private
 
-      def run_loop
-        Thread.new { work_loop }
-      end
-
       def setup
-        setup_configs
-        setup_consumers
-      end
-
-      def work_loop # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        shutdown = false
-
-        while running?
-          break if shutdown
-
-          @consumers.each do |(subscription, config, processor)|
-            break unless running?
-
-            begin
-              @pool.post do
-                timeout = convert_timeout(config[:fetch_timeout])
-                Logger.debug "fetching #{config.dig(:consumer, :subjects).inspect}, timeout=#{timeout}"
-                messages = fetch(subscription, batch_size: config[:batch_size], timeout:)
-                Logger.debug "fetched (#{messages&.size.to_i}) messages"
-                process(messages, processor) if messages&.any?
-                Logger.debug "processed (#{messages&.size.to_i}) messages"
-              end
-            rescue Concurrent::RejectedExecutionError
-              shutdown = true
-              break # pool doesn't accept new jobs, we are shutting down
-            end
+        @configs ||= []
+        @configs = static_config + dynamic_config
 
-            break unless running?
-          end
+        if @options[:processors]
+          pattern = Regexp.new(@options[:processors].map { "\\b#{_1}\\b" }.join("|"))
+          @configs.select! { _1[:class].name.match?(pattern) }
         end
+
+        @configs.each { @consumers << subscribe(nil, _1) }
       end
 
-      def process(messages, processor) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def process(messages, processor) # rubocop:disable Metrics/AbcSize
         metadata = messages.last.metadata
         serializer = processor.class.default_options.dig(:publisher, :serializer)
         messages = messages.map { Message.new(_1, serializer:) }
@@ -70,25 +40,6 @@ module Cosmo
         raise
       end
 
-      def setup_configs
-        @configs = static_config + dynamic_config
-        return unless @options[:processors]
-
-        pattern = Regexp.new(@options[:processors].map { "\\b#{_1}\\b" }.join("|"))
-        @configs.select! { _1[:class].name.match?(pattern) }
-      end
-
-      def setup_consumers
-        @configs.each do |config|
-          processor = config[:class].new
-          subjects = config.dig(:consumer, :subjects)
-          deliver_policy = Config.deliver_policy(config[:start_position])
-          consumer_config, consumer_name = config.values_at(:consumer, :consumer_name)
-          subscription = client.subscribe(subjects, consumer_name, consumer_config.merge(deliver_policy))
-          @consumers << [subscription, config, processor]
-        end
-      end
-
       def static_config
         Config.dig(:consumers, :streams)&.filter_map do |config|
           next unless (klass = Utils::String.safe_constantize(config[:class]))
@@ -98,11 +49,24 @@ module Cosmo
       end
 
       def dynamic_config
-        Config.system[:streams].map { _1.default_options.merge(class: _1) }
+        Config.internal[:streams]&.map { _1.default_options.merge(class: _1) }.to_a
+      end
+
+      def subscribe(_stream_name, config)
+        processor = config[:class].new
+        subjects = config.dig(:consumer, :subjects)
+        deliver_policy = Config.deliver_policy(config[:start_position])
+        consumer_config, consumer_name = config.values_at(:consumer, :consumer_name)
+        subscription = client.subscribe(subjects, consumer_name, consumer_config.merge(deliver_policy))
+        [subscription, config, processor]
+      end
+
+      def fetch_subjects(config)
+        config.dig(:consumer, :subjects)
       end
 
-      def convert_timeout(value)
-        timeout = value.to_f
+      def fetch_timeout(config)
+        timeout = config[:fetch_timeout].to_f
         if timeout <= 0
           Logger.warn "Ignoring `fetch_timeout: #{timeout}` (causes high CPU usage) with #{Data::DEFAULTS[:fetch_timeout]}s instead"
           timeout = Data::DEFAULTS[:fetch_timeout].to_f

data/lib/cosmo/stream.rb

@@ -12,10 +12,11 @@ module Cosmo
     end
 
     module ClassMethods
-      def options(stream: nil, consumer_name: nil, batch_size: nil, fetch_timeout: nil, start_position: nil, consumer: nil, publisher: nil) # rubocop:disable Metrics/ParameterLists
+      def options(stream: nil, consumer_name: nil, batch_size: nil, fetch_timeout: nil, start_position: nil, consumer: nil, publisher: nil)
         register
         default_options.merge!({ stream:, consumer_name:, batch_size:, fetch_timeout:, start_position:, consumer:, publisher: }.compact)
       end
+      alias cosmo_options options
 
       def publish(data, subject: nil, **options)
         stream = default_options[:stream]
@@ -28,8 +29,8 @@ module Cosmo
       end
 
       def register # rubocop:disable Metrics/AbcSize
-        Config.system[:streams] ||= []
-        Config.system[:streams] << self
+        Config.internal[:streams] ||= []
+        Config.internal[:streams] << self
 
         # settings are inherited, don't try to modify them
         return if default_options != Data::DEFAULTS

data/lib/cosmo/utils/hash.rb

@@ -23,20 +23,17 @@ module Cosmo
         end
       end
 
-      # deep dup
-      def dup(hash)
-        Marshal.load(Marshal.dump(hash))
-      end
-
-      # dig keys
-      def keys?(hash, *keys)
-        keys.reduce(hash) do |base, key|
-          return false if !base.is_a?(::Hash) || !base.key?(key)
-
-          base[key]
+      def stringify_keys(obj)
+        case obj
+        when ::Hash
+          obj.each_with_object({}) do |(key, value), result|
+            result[key.to_s] = stringify_keys(value)
+          end
+        when ::Array
+          obj.map { |v| stringify_keys(v) }
+        else
+          obj
         end
-
-        true
       end
 
       # deep set
@@ -49,17 +46,9 @@ module Cosmo
         target[last_key] = value
       end
 
-      # deep merge
-      def merge(hash1, hash2)
-        return hash1 unless hash2
-
-        hash1.merge(hash2) do |_key, old_val, new_val|
-          if old_val.is_a?(::Hash) && new_val.is_a?(::Hash)
-            merge(old_val, new_val)
-          else
-            new_val
-          end
-        end
+      # deep dup
+      def dup(hash)
+        Marshal.load(Marshal.dump(hash))
       end
     end
   end

data/lib/cosmo/utils/overrides.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 Cosmo::Utils::Warnings.silence do
-  members = NATS::JetStream::API::StreamConfig.members + [:allow_msg_counter]
+  members = NATS::JetStream::API::StreamConfig.members + %i[allow_msg_counter allow_msg_schedules]
   NATS::JetStream::API::StreamConfig = Struct.new(*members, keyword_init: true) do
     def initialize(opts = {})
       rem = opts.keys - members

data/lib/cosmo/utils/ttl_cache.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Utils
+    class TTLCache
+      def initialize
+        @store = {}
+      end
+
+      def set(key, value, ttl: nil)
+        @store[key] = [value, ttl ? Time.now + ttl : nil]
+        value
+      end
+
+      def get(key)
+        return unless key?(key)
+
+        @store[key].first
+      end
+
+      def fetch(key, ttl: nil)
+        return get(key) if key?(key)
+
+        result = yield
+        set(key, result, ttl: ttl)
+        result
+      end
+
+      private
+
+      def key?(key)
+        exists = @store.key?(key)
+        return false unless exists
+
+        _, ttl = @store[key]
+        return true unless ttl
+        return true if Time.now < ttl
+
+        @store.delete(key)
+        false
+      end
+    end
+  end
+end

data/lib/cosmo/utils.rb

@@ -7,6 +7,7 @@ require "cosmo/utils/signal"
 require "cosmo/utils/warnings"
 require "cosmo/utils/stopwatch"
 require "cosmo/utils/thread_pool"
+require "cosmo/utils/ttl_cache"
 
 module Cosmo
   module Utils

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

data/lib/cosmo/web/assets/app.css

@@ -135,6 +135,52 @@ button:hover, .btn:hover {
   );
 }
 
+.btn-warning {
+  background: linear-gradient(
+    oklch(from var(--color-warning) calc(l + 0.06) c h),
+    var(--color-warning)
+  );
+  color: oklch(27% 0.005 256);
+}
+.btn-warning:hover {
+  background: linear-gradient(
+    oklch(from var(--color-warning) calc(l + 0.16) c h),
+    oklch(from var(--color-warning) calc(l + 0.1) c h)
+  );
+  color: oklch(27% 0.005 256);
+}
+
+.btn-success {
+  background: linear-gradient(
+    oklch(from var(--color-success) calc(l + 0.06) c h),
+    var(--color-success)
+  );
+}
+.btn-success:hover {
+  background: linear-gradient(
+    oklch(from var(--color-success) calc(l + 0.16) c h),
+    oklch(from var(--color-success) calc(l + 0.1) c h)
+  );
+}
+
+/* ── Badges ────────────────────────────────────────────────────────────── */
+.badge {
+  border-radius: var(--space-1-2);
+  display: inline-block;
+  font-size: var(--font-size-small);
+  font-weight: 700;
+  padding: 2px var(--space);
+  white-space: nowrap;
+}
+.badge-success {
+  background: oklch(from var(--color-success) l c h / 15%);
+  color: var(--color-success);
+}
+.badge-warning {
+  background: oklch(from var(--color-warning) l c h / 20%);
+  color: oklch(from var(--color-warning) calc(l - 0.15) c h);
+}
+
 /* ── Alerts ────────────────────────────────────────────────────────────── */
 .alert {
   padding: var(--space-2x);
@@ -246,6 +292,34 @@ section > header {
   z-index: auto;
 }
 
+/* ── Section tabs ───────────────────────────────────────────────────────── */
+.tabs {
+  display: flex;
+  gap: 0;
+  border-bottom: 2px solid var(--color-border);
+  margin-bottom: var(--space-3x);
+}
+
+.tabs a {
+  padding: var(--space) var(--space-3x);
+  text-decoration: none;
+  color: var(--color-text-light);
+  border-bottom: 2px solid transparent;
+  margin-bottom: -2px;
+  font-weight: 500;
+  transition: color 0.15s, border-color 0.15s;
+}
+
+.tabs a:hover {
+  color: var(--color-text);
+  border-bottom-color: var(--color-border);
+}
+
+.tabs a.active {
+  color: var(--color-primary);
+  border-bottom-color: var(--color-primary);
+}
+
 section .nav {
   display: flex;
   gap: var(--space);
@@ -402,6 +476,20 @@ time {
 .stream-link { color: var(--color-primary); font-weight: 600; }
 .stream-link:hover { text-decoration: underline; }
 
+/* ── Pagination ─────────────────────────────────────────────────────────── */
+.pagination {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: var(--space);
+  padding: var(--space-2x) 0 var(--space);
+}
+.btn-disabled {
+  opacity: 0.4;
+  cursor: default;
+  pointer-events: none;
+}
+
 /* ── Actions ───────────────────────────────────────────────────────────── */
 .actions-form { display: flex; flex-direction: column; gap: var(--space-1-2); }
 

data/lib/cosmo/web/controllers/crons.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "cosmo/web/controllers/application"
+
+module Cosmo
+  class Web
+    module Controllers
+      class Crons < Application
+        def index
+          content_for :title, "Crons"
+          ok render("crons/index", layout: true)
+        end
+
+        def _table
+          ok render("crons/_table", { schedules: cron.all })
+        end
+
+        # Dispatch the job immediately, bypassing the schedule timer.
+        # Expects params["subject"] = the schedule subject stored in NATS.
+        def run_now
+          subject = Rack::Utils.unescape(params["subject"].to_s)
+          cron.run_now!(subject)
+          ok
+        end
+
+        # Purge the schedule from NATS so it stops firing.
+        def delete
+          subject = Rack::Utils.unescape(params["subject"].to_s)
+          cron.delete!(subject)
+          _table
+        end
+
+        private
+
+        def cron
+          @cron ||= API::Cron.instance
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/web/controllers/jobs.rb

@@ -72,12 +72,16 @@ module Cosmo
           ok render("jobs/_busy", { jobs: jobs, total: API::Busy.instance.size })
         end
 
-        def _enqueued
+        def _enqueued # rubocop:disable Metrics/AbcSize
           stream_name, stream_names = streams
+          limit = (params["limit"] || API::Stream::LIMIT).to_i
+          page = [params["page"].to_i, 1].max
           stream = API::Stream.new(stream_name)
-          jobs = stream.messages(page: params["page"], limit: params["limit"])
+          total = stream.total
+          jobs = stream.messages(page:, limit:)
+          total_pages = (total.to_f / limit).ceil
 
-          ok render("jobs/_enqueued", { jobs:, total: stream.total, stream_name:, stream_names: })
+          ok render("jobs/_enqueued", { jobs:, total:, stream_name:, stream_names:, page:, limit:, total_pages: })
         end
 
         def _stats

data/lib/cosmo/web/controllers/streams.rb

@@ -21,22 +21,48 @@ module Cosmo
           ok render("streams/info", { name: name }, layout: true)
         end
 
-        def _table
-          streams = API::Stream.all.map do |stream|
-            state, config = stream.info.values
-            { name: stream.name, messages: state.messages, bytes: state.bytes,
-              first_seq: state.first_seq, last_seq: state.last_seq,
-              consumer_count: state.consumer_count,
-              subjects: config.subjects }
-          end
+        def pause
+          name = Rack::Utils.unescape(@request.params["name"])
+          stream = API::Stream.new(name)
+          stream.pause!
+          return ok render("streams/_pause_banner", banner_locals(stream)) if @request.params["banner"]
+
+          ok render("streams/_stream_row", { stream: row_locals(stream) })
+        end
+
+        def unpause
+          name = Rack::Utils.unescape(@request.params["name"])
+          stream = API::Stream.new(name)
+          stream.unpause!
+          return ok render("streams/_pause_banner", banner_locals(stream)) if @request.params["banner"]
+
+          ok render("streams/_stream_row", { stream: row_locals(stream) })
+        end
 
+        def _table
+          streams = API::Stream.all.reject { |s| s.name.start_with?("KV_") || s.name.start_with?("_cosmo") }.map { row_locals(_1) }
           ok render("streams/_table", { streams: streams })
         end
 
         def _info
           name = Rack::Utils.unescape(@request.params["name"])
-          state = API::Stream.new(name).info.merge(name:)
-          ok render("streams/_info", state)
+          stream = API::Stream.new(name)
+          ok render("streams/_info", stream.info.merge(name:, paused: stream.paused?))
+        end
+
+        private
+
+        def row_locals(stream)
+          state, config = stream.info.values
+          { name: stream.name, messages: state.messages, bytes: state.bytes,
+            first_seq: state.first_seq, last_seq: state.last_seq,
+            consumer_count: state.consumer_count,
+            subjects: config.subjects,
+            paused: stream.paused? }
+        end
+
+        def banner_locals(stream)
+          { name: stream.name, paused: stream.paused? }
         end
       end
     end