cosmonats 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c0995753e33b278f5a0d2d6455faf07f5c3c604595fa93c6a40260c68b69c160
-  data.tar.gz: c78c109d014f863079c3ee53171a852241b5a0b3082f9a29b1438b5708782436
+  metadata.gz: baa2c1820f8c3b420b3ef7411ad74bd8989f5a4d198df6bfe18b5596959d5464
+  data.tar.gz: 282eb6f3dcfc3945596426b3ba39e31cf68d84e73692d0bd92165485b69654ae
 SHA512:
-  metadata.gz: 7947f18a6ab2830fa726d9903eb8ee926bf4bbf897ee71e28c262adc4aa664828c1ad758776aa61caac2964e52c83370cd4d4090c3ce05f399988c79917676e6
-  data.tar.gz: 12a7f6f7e3a715bdda711fa29acbba7db822d17cd4441b008942d7cac0eb66581f2412a4df16811580792af0913c0ffbe1370b0465478e577a6a3a45b714364b
+  metadata.gz: 35c534343db619b2fe9bc314b9517bb67adb808b92c45ae64fc9fb8d3fa533d625d5e1e64149d7ff0ebf8158cd400188c9601ab8b9fe4622f34dbd73dd2a95fc
+  data.tar.gz: '08e9b894165cd0e6254fccca8d73c1cef88bde32e8345de21f812195573add70847c95a0a2cd4c87cb3afc90824028665de2a4a71118aa67c966c056c2680960'

data/lib/cosmo/cli.rb

@@ -15,7 +15,7 @@ module Cosmo
 
     def run
       flags, command, options = parse
-      load_config(flags[:config_file])
+      load_config(flags)
       puts self.class.banner
       boot_application
       require_path(flags[:require])
@@ -37,7 +37,8 @@ module Cosmo
       [flags, command, options]
     end
 
-    def load_config(path)
+    def load_config(flags)
+      path = flags[:config_file]
       raise ConfigNotFoundError, path if path && !File.exist?(path)
 
       unless path
@@ -47,6 +48,8 @@ module Cosmo
       end
 
       Config.load(path)
+      Config.set(:concurrency, flags[:concurrency]) if flags[:concurrency]
+      Config.set(:timeout, flags[:timeout]) if flags[:timeout]
     end
 
     def boot_application
@@ -96,7 +99,8 @@ module Cosmo
         end
 
         o.on "-S", "--setup", "Load config, create streams and exit" do
-          load_config(flags[:config_file])
+          load_config(flags)
+          boot_application
 
           Config[:streams].each do |name, config|
             Client.instance.stream_info(name)

data/lib/cosmo/job/processor.rb

@@ -22,18 +22,23 @@ module Cosmo
           subject = config.delete(:subject)
           priority = config.delete(:priority)
           @weights += ([stream_name] * priority.to_i) if priority
-          @consumers[stream_name] = client.subscribe(subject, consumer_name, config)
+          subscription = client.subscribe(subject, consumer_name, config)
+          @consumers << [subscription, stream_name]
         end
       end
 
-      def work_loop
+      def work_loop # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
         while running?
           @weights.shuffle.each do |stream_name|
             break unless running?
 
             begin
               timeout = ENV.fetch("COSMO_JOBS_FETCH_TIMEOUT", 0.1).to_f
-              @pool.post { fetch_messages(stream_name, batch_size: 1, timeout:) }
+              @pool.post do
+                subscription = @consumers.find { |(_, sn)| sn == stream_name }&.first
+                messages = fetch_messages(subscription, batch_size: 1, timeout:)
+                process(messages) if messages&.any?
+              end
             rescue Concurrent::RejectedExecutionError
               break # pool doesn't accept new jobs, we are shutting down
             end
@@ -43,26 +48,26 @@ module Cosmo
         end
       end
 
-      def schedule_loop # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def schedule_loop # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
         while running?
           break unless running?
 
+          now = Time.now.to_i
           timeout = ENV.fetch("COSMO_JOBS_SCHEDULER_FETCH_TIMEOUT", 5).to_f
-          fetch_messages(:scheduled, batch_size: 100, timeout:) do |messages|
-            now = Time.now.to_i
-            messages.each do |message|
-              headers = message.header.except("X-Stream", "X-Subject", "X-Execute-At", "Nats-Expected-Stream")
-              stream, subject, execute_at = message.header.values_at("X-Stream", "X-Subject", "X-Execute-At")
-              headers["Nats-Expected-Stream"] = stream
-              execute_at = execute_at.to_i
-
-              if now >= execute_at
-                client.publish(subject, message.data, headers: headers)
-                message.ack
-              else
-                delay_ns = (execute_at - now) * 1_000_000_000
-                message.nak(delay: delay_ns)
-              end
+          subscription = @consumers.find { |(_, sn)| sn == :scheduled }&.first
+          messages = fetch_messages(subscription, batch_size: 100, timeout:)
+          messages&.each do |message|
+            headers = message.header.except("X-Stream", "X-Subject", "X-Execute-At", "Nats-Expected-Stream")
+            stream, subject, execute_at = message.header.values_at("X-Stream", "X-Subject", "X-Execute-At")
+            headers["Nats-Expected-Stream"] = stream
+            execute_at = execute_at.to_i
+
+            if now >= execute_at
+              client.publish(subject, message.data, headers: headers)
+              message.ack
+            else
+              delay_ns = (execute_at - now) * 1_000_000_000
+              message.nak(delay: delay_ns)
             end
           end
 

data/lib/cosmo/processor.rb

@@ -10,7 +10,7 @@ module Cosmo
       @pool = pool
       @running = running
       @options = options
-      @consumers = {}
+      @consumers = []
     end
 
     def run
@@ -39,9 +39,8 @@ module Cosmo
       @running.true?
     end
 
-    def fetch_messages(stream_name, batch_size:, timeout:)
-      messages = @consumers[stream_name].fetch(batch_size, timeout:)
-      block_given? ? yield(messages) : process(stream_name, messages)
+    def fetch_messages(subscription, batch_size:, timeout:)
+      subscription.fetch(batch_size, timeout:)
     rescue NATS::Timeout
       # No messages, continue
     end

data/lib/cosmo/stream/processor.rb

@@ -5,8 +5,7 @@ module Cosmo
     class Processor < ::Cosmo::Processor
       def initialize(pool, running, options)
         super
-        @configs = {}
-        @processors = {}
+        @configs = []
       end
 
       private
@@ -17,19 +16,20 @@ module Cosmo
 
       def setup
         setup_configs
-        setup_processors
         setup_consumers
       end
 
-      def work_loop
+      def work_loop # rubocop:disable Metrics/CyclomaticComplexity
         while running?
-          @consumers.each_key do |stream_name|
+          @consumers.each do |(subscription, config, processor)|
             break unless running?
 
             begin
-              batch_size = @configs[stream_name][:batch_size]
               timeout = ENV.fetch("COSMO_STREAMS_FETCH_TIMEOUT", 0.1).to_f
-              @pool.post { fetch_messages(stream_name, batch_size:, timeout:) }
+              @pool.post do
+                messages = fetch_messages(subscription, batch_size: config[:batch_size], timeout:)
+                process(messages, processor) if messages&.any?
+              end
             rescue Concurrent::RejectedExecutionError
               break # pool doesn't accept new jobs, we are shutting down
             end
@@ -39,9 +39,8 @@ module Cosmo
         end
       end
 
-      def process(stream_name, messages) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def process(messages, processor) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         metadata = messages.last.metadata
-        processor = @processors[stream_name]
         serializer = processor.class.default_options.dig(:publisher, :serializer)
         messages = messages.map { Message.new(_1, serializer:) }
 
@@ -64,21 +63,21 @@ module Cosmo
       end
 
       def setup_configs
-        configs = static_config.merge(dynamic_config)
-        configs = configs.select { |_, c| @options[:processors].include?(c[:class].name) } if @options[:processors]
-        @configs.merge!(configs)
-      end
+        @configs = static_config + dynamic_config
+        return unless @options[:processors]
 
-      def setup_processors
-        @configs.each { |s, c| @processors[s] = c[:class].new }
+        pattern = Regexp.new(@options[:processors].map { "\\b#{_1}\\b" }.join("|"))
+        @configs.select! { _1[:class].name.match?(pattern) }
       end
 
       def setup_consumers
-        @configs.each do |stream_name, config|
+        @configs.each do |config|
+          processor = config[:class].new
           subjects = config.dig(:consumer, :subjects)
           deliver_policy = Config.deliver_policy(config[:start_position])
-          config, consumer_name = config.values_at(:consumer, :consumer_name)
-          @consumers[stream_name] = client.subscribe(subjects, consumer_name, config.merge(deliver_policy))
+          consumer_config, consumer_name = config.values_at(:consumer, :consumer_name)
+          subscription = client.subscribe(subjects, consumer_name, consumer_config.merge(deliver_policy))
+          @consumers << [subscription, config, processor]
         end
       end
 
@@ -86,14 +85,12 @@ module Cosmo
         Config.dig(:consumers, :streams)&.filter_map do |config|
           next unless (klass = Utils::String.safe_constantize(config[:class]))
 
-          [config[:stream].to_sym, config.merge(class: klass)]
-        end.to_h
+          config.merge(class: klass)
+        end.to_a
       end
 
       def dynamic_config
-        Config.system[:streams].to_h do |klass|
-          [klass.default_options[:stream].to_sym, klass.default_options.merge(class: klass)]
-        end
+        Config.system[:streams].map { _1.default_options.merge(class: _1) }
       end
     end
   end

data/lib/cosmo/stream.rb

@@ -9,12 +9,12 @@ module Cosmo
   module Stream
     def self.included(base)
       base.extend(ClassMethods)
-      base.register
     end
 
     module ClassMethods
       def options(stream: nil, consumer_name: nil, batch_size: nil, start_position: nil, consumer: nil, publisher: nil) # rubocop:disable Metrics/ParameterLists
-        default_options.merge!({ stream: stream, consumer_name: consumer_name, batch_size:, start_position:, consumer:, publisher: }.compact)
+        register
+        default_options.merge!({ stream:, consumer_name:, batch_size:, start_position:, consumer:, publisher: }.compact)
       end
 
       def publish(data, subject: nil, **options)

data/lib/cosmo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cosmo
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/sig/cosmo/cli.rbs

@@ -14,7 +14,7 @@ module Cosmo
 
     def parse: () -> [Hash[Symbol, untyped], ::String?, Hash[Symbol, untyped]]
 
-    def load_config: (::String? path) -> void
+    def load_config: (Hash[Symbol, untyped] flags) -> void
 
     def boot_application: () -> void
 

data/sig/cosmo/job/processor.rbs

@@ -3,7 +3,7 @@ module Cosmo
     class Processor < ::Cosmo::Processor
       @weights: Array[Symbol]
 
-      def initialize: (Utils::ThreadPool pool, untyped running) -> void
+      def initialize: (Utils::ThreadPool pool, untyped running, Hash[Symbol, untyped] options) -> void
 
       private
 

data/sig/cosmo/processor.rbs

@@ -2,7 +2,7 @@ module Cosmo
   class Processor
     @pool: Utils::ThreadPool
     @running: untyped
-    @consumers: Hash[Symbol, untyped]
+    @consumers: Array[untyped]
     @options: Hash[Symbol, untyped]
 
     def self.run: (*untyped) -> Processor
@@ -21,7 +21,7 @@ module Cosmo
 
     def running?: () -> bool
 
-    def fetch_messages: (Symbol stream_name, batch_size: Integer, timeout: Float) ?{ (Array[untyped]) -> void } -> void
+    def fetch_messages: (untyped subscription, batch_size: Integer, timeout: Float) ?{ (Array[untyped]) -> void } -> void
 
     def client: () -> Client
 

data/sig/cosmo/stream/processor.rbs

@@ -1,10 +1,9 @@
 module Cosmo
   module Stream
     class Processor < ::Cosmo::Processor
-      @configs: Hash[Symbol, Hash[Symbol, untyped]]
-      @processors: Hash[Symbol, untyped]
+      @configs: Array[Hash[Symbol, untyped]]
 
-      def initialize: (Utils::ThreadPool pool, untyped running) -> void
+      def initialize: (Utils::ThreadPool pool, untyped running, Hash[Symbol, untyped] options) -> void
 
       private
 
@@ -14,17 +13,15 @@ module Cosmo
 
       def work_loop: () -> void
 
-      def process: (Symbol stream_name, Array[untyped] messages) -> void
+      def process: (Array[untyped] messages, untyped processor) -> void
 
       def setup_configs: () -> void
 
-      def setup_processors: () -> void
-
       def setup_consumers: () -> void
 
-      def static_config: -> Hash[Symbol, untyped]
+      def static_config: -> Array[Hash[Symbol, untyped]]
 
-      def dynamic_config: -> Hash[Symbol, untyped]
+      def dynamic_config: -> Array[Hash[Symbol, untyped]]
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cosmonats
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Dmitry Vorotilin