cosmonats 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c5a169718d8f3ddb85e2fcc95a5e1b6c0fd3061c1ad4410bd7bad85add934c6
-  data.tar.gz: aa9715564bdbead26f876ff9de8c9119084015fb3e0d52e2ec6b126c3d3ad0a0
+  metadata.gz: c001e33a420b73f46eebfe6ecffca697ef7d67f0436a2b70a88ed6207b3846a5
+  data.tar.gz: 2fd541b16a24b5a461ea60078be20d92a0c815d0a09996fc9e419f958fce31ea
 SHA512:
-  metadata.gz: 41bf7cc2664b566e0f70f9b4045a921abe7afc0e0cd33897f46dd975d4148aead93eaf7faeaffd7364268acee1713b08daeb151c7b124412de2983970787f97b
-  data.tar.gz: 761a89e2379601c71b75e6c13f63e0c7c0dda9fa7f5631aa6b0f8bab7493f3fe29bee1e562677be8f22ac45b1597f4a0f34660b91e999a7eceb7367d0db50d33
+  metadata.gz: b401aaac7b86f8d08167d256814bed8cd8332aa1f9d82e97548f89d5f3770f95d90a8c36dc711eb0798fbc21ec654d21f629879e3fb56e8d3ecc4c943411be24
+  data.tar.gz: 8ff68d161c0f1f15dabddab8e36ad37363456126c5332bdfe636620e68c91510d690048585fa2064538e77ef3be6c1f892be4f57878de844d9f32d4a46815da3

data/lib/cosmo/client.rb

@@ -11,7 +11,9 @@ module Cosmo
     attr_reader :nc, :js
 
     def initialize(nats_url: ENV.fetch("NATS_URL", "nats://localhost:4222"))
+      Logger.debug "Connecting to NATS server at #{nats_url}..."
       @nc = NATS.connect(nats_url)
+      Logger.debug "Connection established"
       @js = @nc.jetstream
     end
 

data/lib/cosmo/job/processor.rb

@@ -28,7 +28,11 @@ module Cosmo
       end
 
       def work_loop # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
+        shutdown = false
+
         while running?
+          break if shutdown
+
           @weights.shuffle.each do |stream_name|
             break unless running?
 
@@ -36,10 +40,11 @@ module Cosmo
               timeout = ENV.fetch("COSMO_JOBS_FETCH_TIMEOUT", 0.1).to_f
               @pool.post do
                 subscription = @consumers.find { |(_, sn)| sn == stream_name }&.first
-                messages = fetch_messages(subscription, batch_size: 1, timeout:)
+                messages = fetch(subscription, batch_size: 1, timeout:)
                 process(messages) if messages&.any?
               end
             rescue Concurrent::RejectedExecutionError
+              shutdown = true
               break # pool doesn't accept new jobs, we are shutting down
             end
 
@@ -55,7 +60,7 @@ module Cosmo
           now = Time.now.to_i
           timeout = ENV.fetch("COSMO_JOBS_SCHEDULER_FETCH_TIMEOUT", 5).to_f
           subscription = @consumers.find { |(_, sn)| sn == :scheduled }&.first
-          messages = fetch_messages(subscription, batch_size: 100, timeout:)
+          messages = fetch(subscription, batch_size: 100, timeout:)
           messages&.each do |message|
             headers = message.header.except("X-Stream", "X-Subject", "X-Execute-At", "Nats-Expected-Stream")
             stream, subject, execute_at = message.header.values_at("X-Stream", "X-Subject", "X-Execute-At")

data/lib/cosmo/processor.rb

@@ -39,10 +39,16 @@ module Cosmo
       @running.true?
     end
 
-    def fetch_messages(subscription, batch_size:, timeout:)
+    def fetch(subscription, batch_size:, timeout:)
       subscription.fetch(batch_size, timeout:)
     rescue NATS::Timeout
       # No messages, continue
+    rescue StandardError => e
+      Logger.error "Snap! Error just happened"
+      Logger.error "#{e.class}: #{e.message}"
+
+      backoff = ENV.fetch("COSMO_STREAMS_FETCH_BACKOFF", 5).to_f
+      sleep([timeout, backoff].max) # backoff before retry
     end
 
     def client

data/lib/cosmo/stream/data.rb

@@ -7,6 +7,7 @@ module Cosmo
     class Data
       DEFAULTS = {
         batch_size: 100,
+        fetch_timeout: 10.0,
         consumer: {
           ack_policy: "explicit",
           max_deliver: 1,

data/lib/cosmo/stream/processor.rb

@@ -19,21 +19,26 @@ module Cosmo
         setup_consumers
       end
 
-      def work_loop # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
+      def work_loop # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        shutdown = false
+
         while running?
+          break if shutdown
+
           @consumers.each do |(subscription, config, processor)|
             break unless running?
 
             begin
-              timeout = ENV.fetch("COSMO_STREAMS_FETCH_TIMEOUT", 0.1).to_f
               @pool.post do
+                timeout = convert_timeout(config[:fetch_timeout])
                 Logger.debug "fetching #{config.dig(:consumer, :subjects).inspect}, timeout=#{timeout}"
-                messages = fetch_messages(subscription, batch_size: config[:batch_size], timeout:)
+                messages = fetch(subscription, batch_size: config[:batch_size], timeout:)
                 Logger.debug "fetched (#{messages&.size.to_i}) messages"
                 process(messages, processor) if messages&.any?
                 Logger.debug "processed (#{messages&.size.to_i}) messages"
               end
             rescue Concurrent::RejectedExecutionError
+              shutdown = true
               break # pool doesn't accept new jobs, we are shutting down
             end
 
@@ -95,6 +100,16 @@ module Cosmo
       def dynamic_config
         Config.system[:streams].map { _1.default_options.merge(class: _1) }
       end
+
+      def convert_timeout(value)
+        timeout = value.to_f
+        if timeout <= 0
+          Logger.warn "Ignoring `fetch_timeout: #{timeout}` (causes high CPU usage) with #{Data::DEFAULTS[:fetch_timeout]}s instead"
+          timeout = Data::DEFAULTS[:fetch_timeout].to_f
+        end
+
+        timeout
+      end
     end
   end
 end

data/lib/cosmo/stream.rb

@@ -12,9 +12,9 @@ module Cosmo
     end
 
     module ClassMethods
-      def options(stream: nil, consumer_name: nil, batch_size: nil, start_position: nil, consumer: nil, publisher: nil) # rubocop:disable Metrics/ParameterLists
+      def options(stream: nil, consumer_name: nil, batch_size: nil, fetch_timeout: nil, start_position: nil, consumer: nil, publisher: nil) # rubocop:disable Metrics/ParameterLists
         register
-        default_options.merge!({ stream:, consumer_name:, batch_size:, start_position:, consumer:, publisher: }.compact)
+        default_options.merge!({ stream:, consumer_name:, batch_size:, fetch_timeout:, start_position:, consumer:, publisher: }.compact)
       end
 
       def publish(data, subject: nil, **options)

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/sig/cosmo/processor.rbs

@@ -21,7 +21,7 @@ module Cosmo
 
     def running?: () -> bool
 
-    def fetch_messages: (untyped subscription, batch_size: Integer, timeout: Float) ?{ (Array[untyped]) -> void } -> void
+    def fetch: (untyped subscription, batch_size: Integer, timeout: Float) ?{ (Array[untyped]) -> void } -> void
 
     def client: () -> Client
 

data/sig/cosmo/stream/data.rbs

@@ -1,7 +1,7 @@
 module Cosmo
   module Stream
     class Data
-      DEFAULTS: Hash[Symbol, Integer | Hash[Symbol, untyped]]
+      DEFAULTS: Hash[Symbol, Numeric | Hash[Symbol, untyped]]
     end
   end
 end

data/sig/cosmo/stream/processor.rbs

@@ -11,6 +11,8 @@ module Cosmo
 
       def setup: () -> void
 
+      def convert_timeout: (Numeric) -> Float
+
       def work_loop: () -> void
 
       def process: (Array[untyped] messages, untyped processor) -> void

data/sig/cosmo/stream.rbs

@@ -9,6 +9,7 @@ module Cosmo
         ?stream: Symbol?,
         ?consumer_name: ::String?,
         ?batch_size: Integer?,
+        ?fetch_timeout: (Integer | Float)?,
         ?start_position: (Symbol | ::String | Time)?,
         ?consumer: Hash[Symbol, untyped]?,
         ?publisher: Hash[Symbol, untyped]?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cosmonats
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Dmitry Vorotilin
@@ -82,11 +82,11 @@ files:
 - sig/cosmo/job/data.rbs
 - sig/cosmo/job/processor.rbs
 - sig/cosmo/logger.rbs
-- sig/cosmo/message.rbs
 - sig/cosmo/processor.rbs
 - sig/cosmo/publisher.rbs
 - sig/cosmo/stream.rbs
 - sig/cosmo/stream/data.rbs
+- sig/cosmo/stream/message.rbs
 - sig/cosmo/stream/processor.rbs
 - sig/cosmo/stream/serializer.rbs
 - sig/cosmo/utils/hash.rbs