cosmonats 0.1.4 → 0.3.0

data/lib/cosmo/processor.rb

@@ -1,16 +1,24 @@
 # frozen_string_literal: true
 
 module Cosmo
-  class Processor
+  class Processor # rubocop:disable Metrics/ClassLength
+    STREAM_PAUSED_RECHECK_TTL = 5.0 # Seconds a stream's paused state is cached before re-checking (override via COSMO_STREAM_PAUSED_RECHECK_TTL)
+    STREAMS_PAUSED_IDLE_SLEEP = 1.0 # Seconds to sleep when every stream is paused, preventing a tight CPU spin (override via COSMO_STREAMS_PAUSED_IDLE_SLEEP)
+    STREAM_EMPTY_BACKOFF_MAX = 5.0  # Max seconds to sleep between empty fetches (override via COSMO_STREAM_EMPTY_BACKOFF_MAX)
+
     def self.run(...)
       new(...).tap(&:run)
     end
 
+    attr_reader :consumers
+
     def initialize(pool, running, options)
       @pool = pool
       @running = running
       @options = options
+      @threads = []
       @consumers = []
+      @cache = Utils::TTLCache.new
     end
 
     def run
@@ -21,9 +29,95 @@ module Cosmo
       run_loop
     end
 
+    def stop(timeout = Config[:timeout])
+      @running.make_false
+      @pool.shutdown
+      @consumers.each { |(s, _)| s.unsubscribe rescue nil }
+      @pool.wait_for_termination(timeout)
+      @threads.compact.each { _1.join(timeout) || _1.kill }
+      @consumers.clear
+      @threads.clear
+    end
+
     private
 
     def run_loop
+      @threads << Thread.new { work_loop }
+      @threads << Thread.new { schedule_loop } if scheduler?
+    end
+
+    def work_loop # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      shutdown = false
+
+      while running?
+        break if shutdown
+
+        all_empty = true  # every stream is empty
+        all_paused = true # every stream is paused
+        consumers.each do |(subscription, config, processor)| # rubocop:disable Metrics/BlockLength
+          break unless running?
+
+          stream_name = config[:stream].to_s
+          ttl = ENV.fetch("COSMO_STREAM_PAUSED_RECHECK_TTL", STREAM_PAUSED_RECHECK_TTL).to_f
+          if @cache.fetch(stream_name, ttl:) { API::Stream.new(stream_name).paused? }
+            Logger.debug "stream #{stream_name} is paused, skipping fetch"
+            next
+          end
+          all_paused = false
+
+          _, skip_t = consumer_state[stream_name]
+          if skip_t && Time.now < skip_t
+            Logger.debug "stream #{stream_name} is empty, backing off"
+            next
+          end
+          all_empty = false
+
+          begin
+            @pool.post do
+              # Re-check, after possibly being blocked on post to thread pool
+              _, skip_t = consumer_state[stream_name]
+              next if skip_t && Time.now < skip_t
+
+              timeout = fetch_timeout(config)
+              Logger.debug "fetching #{fetch_subjects(config).inspect}, timeout=#{timeout}"
+              messages = lock(stream_name) { fetch(subscription, batch_size: config[:batch_size], timeout:) }
+              Logger.debug "fetched (#{messages&.size.to_i}) messages"
+              if messages&.any?
+                consumer_state.delete(stream_name)
+                process(messages, processor)
+              else
+                max_backoff = ENV.fetch("COSMO_STREAM_EMPTY_BACKOFF_MAX", STREAM_EMPTY_BACKOFF_MAX).to_f
+                consumer_state.compute(stream_name) do |current|
+                  count = (current&.first || 0) + 1
+                  backoff = [timeout * (2**(count - 1)), max_backoff].min
+                  [count, Time.now + backoff]
+                end
+              end
+            end
+          rescue Concurrent::RejectedExecutionError
+            shutdown = true
+            break # pool doesn't accept new jobs, we are shutting down
+          end
+        end
+
+        break unless running?
+
+        if all_paused
+          period = ENV.fetch("COSMO_STREAMS_PAUSED_IDLE_SLEEP", STREAMS_PAUSED_IDLE_SLEEP).to_f
+          Logger.debug "all streams paused, sleep=#{period}"
+          sleep(period)
+        elsif all_empty
+          next_wake = consumer_state.values.filter_map { |_, t| t }.min
+          next unless next_wake # entry was deleted concurrently (messages arrived), re-loop immediately
+
+          remaining = [next_wake - Time.now, 0.01].max
+          Logger.debug "all streams empty, sleep=#{remaining}"
+          sleep(remaining)
+        end
+      end
+    end
+
+    def schedule_loop
       raise NotImplementedError
     end
 
@@ -39,16 +133,21 @@ module Cosmo
       @running.true?
     end
 
+    def scheduler?
+      false
+    end
+
     def fetch(subscription, batch_size:, timeout:)
       subscription.fetch(batch_size, timeout:)
     rescue NATS::Timeout
       # No messages, continue
     rescue StandardError => e
       Logger.error "Snap! Error just happened"
-      Logger.error "#{e.class}: #{e.message}"
+      Logger.error "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
 
       backoff = ENV.fetch("COSMO_STREAMS_FETCH_BACKOFF", 5).to_f
       sleep([timeout, backoff].max) # backoff before retry
+      nil
     end
 
     def client
@@ -58,5 +157,14 @@ module Cosmo
     def stopwatch
       Utils::Stopwatch.new
     end
+
+    def lock(stream_name, &)
+      @locks ||= Hash.new { |h, k| h[k] = Mutex.new }
+      @locks[stream_name].synchronize(&)
+    end
+
+    def consumer_state
+      @consumer_state ||= Concurrent::Map.new
+    end
   end
 end

data/lib/cosmo/stream/processor.rb

@@ -3,48 +3,18 @@
 module Cosmo
   module Stream
     class Processor < ::Cosmo::Processor
-      def initialize(pool, running, options)
-        super
-        @configs = []
-      end
-
       private
 
-      def run_loop
-        Thread.new { work_loop }
-      end
-
       def setup
-        setup_configs
-        setup_consumers
-      end
-
-      def work_loop # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-        shutdown = false
-
-        while running?
-          break if shutdown
-
-          @consumers.each do |(subscription, config, processor)|
-            break unless running?
-
-            begin
-              @pool.post do
-                timeout = convert_timeout(config[:fetch_timeout])
-                Logger.debug "fetching #{config.dig(:consumer, :subjects).inspect}, timeout=#{timeout}"
-                messages = fetch(subscription, batch_size: config[:batch_size], timeout:)
-                Logger.debug "fetched (#{messages&.size.to_i}) messages"
-                process(messages, processor) if messages&.any?
-                Logger.debug "processed (#{messages&.size.to_i}) messages"
-              end
-            rescue Concurrent::RejectedExecutionError
-              shutdown = true
-              break # pool doesn't accept new jobs, we are shutting down
-            end
+        @configs ||= []
+        @configs = static_config + dynamic_config
 
-            break unless running?
-          end
+        if @options[:processors]
+          pattern = Regexp.new(@options[:processors].map { "\\b#{_1}\\b" }.join("|"))
+          @configs.select! { _1[:class].name.match?(pattern) }
         end
+
+        @configs.each { @consumers << subscribe(nil, _1) }
       end
 
       def process(messages, processor) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
@@ -70,25 +40,6 @@ module Cosmo
         raise
       end
 
-      def setup_configs
-        @configs = static_config + dynamic_config
-        return unless @options[:processors]
-
-        pattern = Regexp.new(@options[:processors].map { "\\b#{_1}\\b" }.join("|"))
-        @configs.select! { _1[:class].name.match?(pattern) }
-      end
-
-      def setup_consumers
-        @configs.each do |config|
-          processor = config[:class].new
-          subjects = config.dig(:consumer, :subjects)
-          deliver_policy = Config.deliver_policy(config[:start_position])
-          consumer_config, consumer_name = config.values_at(:consumer, :consumer_name)
-          subscription = client.subscribe(subjects, consumer_name, consumer_config.merge(deliver_policy))
-          @consumers << [subscription, config, processor]
-        end
-      end
-
       def static_config
         Config.dig(:consumers, :streams)&.filter_map do |config|
           next unless (klass = Utils::String.safe_constantize(config[:class]))
@@ -98,11 +49,24 @@ module Cosmo
       end
 
       def dynamic_config
-        Config.system[:streams].map { _1.default_options.merge(class: _1) }
+        Config.internal[:streams].map { _1.default_options.merge(class: _1) }
+      end
+
+      def subscribe(_stream_name, config)
+        processor = config[:class].new
+        subjects = config.dig(:consumer, :subjects)
+        deliver_policy = Config.deliver_policy(config[:start_position])
+        consumer_config, consumer_name = config.values_at(:consumer, :consumer_name)
+        subscription = client.subscribe(subjects, consumer_name, consumer_config.merge(deliver_policy))
+        [subscription, config, processor]
+      end
+
+      def fetch_subjects(config)
+        config.dig(:consumer, :subjects)
       end
 
-      def convert_timeout(value)
-        timeout = value.to_f
+      def fetch_timeout(config)
+        timeout = config[:fetch_timeout].to_f
         if timeout <= 0
           Logger.warn "Ignoring `fetch_timeout: #{timeout}` (causes high CPU usage) with #{Data::DEFAULTS[:fetch_timeout]}s instead"
           timeout = Data::DEFAULTS[:fetch_timeout].to_f

data/lib/cosmo/stream.rb

@@ -28,8 +28,8 @@ module Cosmo
       end
 
       def register # rubocop:disable Metrics/AbcSize
-        Config.system[:streams] ||= []
-        Config.system[:streams] << self
+        Config.internal[:streams] ||= []
+        Config.internal[:streams] << self
 
         # settings are inherited, don't try to modify them
         return if default_options != Data::DEFAULTS

data/lib/cosmo/utils/hash.rb

@@ -23,22 +23,6 @@ module Cosmo
         end
       end
 
-      # deep dup
-      def dup(hash)
-        Marshal.load(Marshal.dump(hash))
-      end
-
-      # dig keys
-      def keys?(hash, *keys)
-        keys.reduce(hash) do |base, key|
-          return false if !base.is_a?(::Hash) || !base.key?(key)
-
-          base[key]
-        end
-
-        true
-      end
-
       # deep set
       def set(hash, *keys, value)
         last_key = keys.pop
@@ -49,17 +33,9 @@ module Cosmo
         target[last_key] = value
       end
 
-      # deep merge
-      def merge(hash1, hash2)
-        return hash1 unless hash2
-
-        hash1.merge(hash2) do |_key, old_val, new_val|
-          if old_val.is_a?(::Hash) && new_val.is_a?(::Hash)
-            merge(old_val, new_val)
-          else
-            new_val
-          end
-        end
+      # deep dup
+      def dup(hash)
+        Marshal.load(Marshal.dump(hash))
       end
     end
   end

data/lib/cosmo/utils/overrides.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+Cosmo::Utils::Warnings.silence do
+  members = NATS::JetStream::API::StreamConfig.members + [:allow_msg_counter]
+  NATS::JetStream::API::StreamConfig = Struct.new(*members, keyword_init: true) do
+    def initialize(opts = {})
+      rem = opts.keys - members
+      opts.delete_if { |k| rem.include?(k) }
+      super
+    end
+  end
+
+  members = NATS::JetStream::PubAck.members + [:val]
+  NATS::JetStream::PubAck = Struct.new(*members, keyword_init: true)
+end

data/lib/cosmo/utils/ttl_cache.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Utils
+    class TTLCache
+      def initialize
+        @store = {}
+      end
+
+      def set(key, value, ttl: nil)
+        @store[key] = [value, ttl ? Time.now + ttl : nil]
+        value
+      end
+
+      def get(key)
+        return unless key?(key)
+
+        @store[key].first
+      end
+
+      def fetch(key, ttl: nil)
+        return get(key) if key?(key)
+
+        result = yield
+        set(key, result, ttl: ttl)
+        result
+      end
+
+      private
+
+      def key?(key)
+        exists = @store.key?(key)
+        return false unless exists
+
+        _, ttl = @store[key]
+        return true unless ttl
+        return true if Time.now < ttl
+
+        @store.delete(key)
+        false
+      end
+    end
+  end
+end

data/lib/cosmo/utils/warnings.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Utils
+    module Warnings
+      module_function
+
+      def silence
+        verbose = $VERBOSE
+        $VERBOSE = nil
+        yield
+      ensure
+        $VERBOSE = verbose
+      end
+    end
+  end
+end

data/lib/cosmo/utils.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "cosmo/utils/hash"
+require "cosmo/utils/json"
+require "cosmo/utils/string"
+require "cosmo/utils/signal"
+require "cosmo/utils/warnings"
+require "cosmo/utils/stopwatch"
+require "cosmo/utils/thread_pool"
+require "cosmo/utils/ttl_cache"
+
+module Cosmo
+  module Utils
+  end
+end

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.1.4"
+  VERSION = "0.3.0"
 end