cosmonats 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b63356c69b61ea32b4791b519830003e4f42330d39947a332cfbfdb20ed91c7d
-  data.tar.gz: 6cc0401b58b06038dcc4dc7e4bea9ff695662fa1b251b89fd681a66e893387c7
+  metadata.gz: f850638b66506538b131180c03b64a13e31cd86f924503703812da49b6ba3d20
+  data.tar.gz: 0dfdab08193600f3bca2c32558ffb016a38657f9488789770eea5e60612281e9
 SHA512:
-  metadata.gz: 5c7ef63abf649154cb9a5e5662e29297cd4b171810f819b16401f08651a5ee3a8c3111421ed3873ea4c12477d05cd6329e18f67feda91edadf7605eadf2a4b91
-  data.tar.gz: d1bb560e31a2c0fb6c321d52599a543a372f184e136f29778a44451648eaa7beed1405d83d1fe996f87b8ee029432d8f395c3fad348a7eaacb73de958ad31573
+  metadata.gz: 56f46ac8a8c1409b1583033ad8b5cf9e513652fae2643b07344aa8f8bf1cca80f948463af92e2a989ec9679cc830ac05a6b873b1f0c9b5e10a1e6ef5e8399ad0
+  data.tar.gz: '086ae5c5b26471417e54c37d2f782c0c103340953e9a27ca8c02131478122f01ede30f2f8ab5a0b29c81ce7b4ea07a5c8de8e7f51957e4d9d5dfd5d670a0ff08'

data/README.md

@@ -95,62 +95,69 @@ gem "cosmonats"
 
 **Requirements:** Ruby 3.1.0+, NATS Server ([installation guide](https://docs.nats.io/running-a-nats-service/introduction/installation))
 
+Add these lines to config/routes.rb:
+```ruby
+require "cosmo/web"
+
+Rails.application.routes.draw do
+  mount Cosmo::Web => "/cosmo" # access web UI at http://localhost:3000/cosmo
+  ...
+end
+```
+
 
 ## 🚀 Quick Start
 
-### 1. Create a Job
+### 1. Create `config/cosmo.yml` and run `bundle exec cosmo -S` to create streams in NATS:
+
+```yaml
+concurrency: 5
+max_retries: 3
+
+consumers:
+  jobs:
+    default:
+      ack_policy: explicit
+      max_deliver: 10
+      max_ack_pending: 10
+      ack_wait: 15
+      subject: jobs.%{name}.>
+
+setup:
+  jobs:
+    default:
+      storage: file
+      retention: workqueue
+      subjects: ["jobs.%{name}.>"]
+      allow_direct: true
+``` 
+
+### 2. Create a Job in app/workers
 
 ```ruby
 class SendEmailJob
   include Cosmo::Job
 
-  # configure job options (optional)
   options stream: :default, retry: 3, dead: true
 
   def perform(user_id, email_type)
-    user = User.find(user_id)
-    UserMailer.send(email_type, user).deliver_now
+    # Pretend to send email to user: UserMailer.send(email_type, user_id).deliver_now
+    sleep 0.5 # Simulate work
+    puts "#{user_id}, #{email_type}"
   end
 end
 ```
 
-### 2. Enqueue Jobs
+### 3. Enqueue Jobs
 
 ```ruby
-SendEmailJob.perform_async(123, 'welcome')           # Immediately
-SendEmailJob.perform_in(1.hour, 123, 'reminder')     # Delayed
-SendEmailJob.perform_at(1.day.from_now, 123, 'test') # Scheduled
+10.times { |i| SendEmailJob.perform_async(i, "welcome") }
 ```
 
-### 3. Configure (config/cosmo.yml)
-
-```yaml
-concurrency: 10
-max_retries: 3
-
-consumers:
-  jobs:
-    default:
-      ack_policy: explicit
-      max_deliver: 3
-      max_ack_pending: 3
-      ack_wait: 60
-
-streams:
-  default:
-    storage: file
-    retention: workqueue
-    subjects: ["jobs.default.>"]
-```
-
-### 4. Setup & Run
+### 4. Run
 
 ```bash
-# Setup streams
-cosmo -C config/cosmo.yml --setup
-
-# Start processing
-cosmo -C config/cosmo.yml -c 10 -r ./app/jobs jobs
+bundle exec cosmo jobs
 ```
 
 
@@ -182,7 +189,7 @@ end
 # Usage
 ReportJob.perform_async(42)                              # Enqueue now
 ReportJob.perform_in(30.minutes, 42)                     # Delayed
-ReportJob.perform_at(Time.parse('2026-01-25 10:00'), 42) # Scheduled
+ReportJob.perform_at(Time.parse("2026-01-25 10:00"), 42) # Scheduled
 ```
 
 ### Streams
@@ -221,8 +228,8 @@ end
 
 # Publishing
 ClicksProcessor.publish(
-  { user_id: 123, page: '/home' },
-  subject: 'events.clicks.homepage'
+  { user_id: 123, page: "/home" },
+  subject: "events.clicks.homepage"
 )
 
 # Message acknowledgment strategies
@@ -235,32 +242,86 @@ message.term                         # Permanent failure, no retry
 
 **File-based (config/cosmo.yml):**
 ```yaml
-timeout: 25     # Shutdown timeout in seconds
-concurrency: 10 # Number of worker threads
-max_retries: 3  # Default max retries
+timeout: 25                 # Shutdown timeout in seconds
+concurrency: &concurrency 1 # Number of worker threads
+max_retries: &max_retries 3 # Default max retries
+
+stream_config: &stream_config
+  storage: file         # storage type (file or memory)
+  retention: workqueue  # retention policy (limits, interest, workqueue)
+  duplicate_window: 120 # time window for duplicate message detection in seconds
+  discard: old          # discard new messages when stream is full (discard new or old)
+  allow_direct: true    # allow direct messages to stream, required for web UI
+  subjects:
+    - jobs.%{name}.>    # subject pattern for stream, %{name} will be replaced with stream name
+
+consumer_config: &consumer_config
+  ack_policy: explicit    # ack policy (explicit, none, all), each individual message must be acknowledged
+  max_deliver: 10         # maximum number of times a message will be delivered before it's considered failed
+  max_ack_pending: 20     # maximum number of messages with pending ack for this consumer
+  ack_wait: 60            # time in seconds to wait for an ack before redelivering the message
+  subject: jobs.%{name}.> # subject pattern for consumer, %{name} will be replaced with stream name
 
 consumers:
-  streams:
-    - class: MyStream
-      batch_size: 50
-      consumer:
-        ack_policy: explicit
-        max_deliver: 3
-        subjects: ["events.>"]
+  jobs:
+    critical:
+      <<: *consumer_config
+      priority: 50
+    high:
+      <<: *consumer_config
+      priority: 30
+    default:
+      <<: *consumer_config
+      priority: 15
+    low:
+      <<: *consumer_config
+      priority: 5
+    scheduled:
+      <<: *consumer_config
+      max_deliver: 1
+      max_ack_pending: 100
+      ack_wait: 10
 
 setup:
-  streams:
-    my_stream:
-      storage: file         # or memory
-      retention: workqueue  # or limits
-      max_age: 86400       # 1d in seconds
-      subjects: ["events.>"]
+  jobs:
+    critical:
+      <<: *stream_config
+      description: Very critical priority jobs
+    high:
+      <<: *stream_config
+      description: Higher priority jobs
+    default:
+      <<: *stream_config
+      description: Default priority jobs
+    low:
+      <<: *stream_config
+      description: Lower priority jobs
+    scheduled:
+      <<: *stream_config
+      description: Scheduled jobs
+    dead:
+      <<: *stream_config
+      retention: limits
+      max_msgs: 10000
+      max_age: 604800 # 7d
+      description: Broken jobs (DLQ)
+
+development:
+  verbose: false
+  concurrency: *concurrency
+
+staging:
+  verbose: true
+  concurrency: 3
+
+production:
+  concurrency: 3
 ```
 
 **Programmatic:**
 ```ruby
 Cosmo::Config.set(:concurrency, 20)
-Cosmo::Config.set(:setup, :streams, :custom, { storage: 'file', subjects: ['custom.>'] })
+Cosmo::Config.set(:setup, :streams, :custom, { storage: "file", subjects: ["custom.>"] })
 ```
 
 **Environment variables:**
@@ -328,10 +389,10 @@ end
 **Testing:**
 ```ruby
 # Synchronous execution
-SendEmailJob.perform_sync(123, 'test')
+SendEmailJob.perform_sync(123, "test")
 
 # Test job creation
-jid = SendEmailJob.perform_async(123, 'welcome')
+jid = SendEmailJob.perform_async(123, "welcome")
 assert_kind_of String, jid
 ```
 
@@ -438,7 +499,7 @@ sudo systemctl status cosmo
 **Stream Metrics:**
 ```ruby
 client = Cosmo::Client.instance
-info = client.stream_info('default')
+info = client.stream_info("default")
 
 info.state.messages       # Total messages
 info.state.bytes          # Total bytes
@@ -465,8 +526,8 @@ class EmailJob
   end
 end
 
-EmailJob.perform_async(123, 'welcome')
-EmailJob.perform_in(1.day, 123, 'followup')
+EmailJob.perform_async(123, "welcome")
+EmailJob.perform_in(1.day, 123, "followup")
 ```
 
 **Image Processing Pipeline:**
@@ -475,12 +536,12 @@ class ImageProcessor
   include Cosmo::Stream
   options(
     stream: :images,
-    consumer: { subjects: ['images.uploaded.>'] }
+    consumer: { subjects: ["images.uploaded.>"] }
   )
 
   def process_one
-    processed = ImageService.process(message.data['url'])
-    publish(processed, subject: 'images.processed.optimized')
+    processed = ImageService.process(message.data["url"])
+    publish(processed, subject: "images.processed.optimized")
     message.ack
   rescue => e
     logger.error "Processing failed: #{e.message}"
@@ -488,18 +549,18 @@ class ImageProcessor
   end
 end
 
-ImageProcessor.publish({ url: 'https://example.com/image.jpg' }, subject: 'images.uploaded.user')
+ImageProcessor.publish({ url: "https://example.com/image.jpg" }, subject: "images.uploaded.user")
 ```
 
 **Real-Time Analytics:**
 ```ruby
 class AnalyticsAggregator
   include Cosmo::Stream
-  options batch_size: 1000, consumer: { subjects: ['events.*.>'] }
+  options batch_size: 1000, consumer: { subjects: ["events.*.>"] }
 
   def process(messages)
     events = messages.map(&:data)
-    aggregates = events.group_by { |e| e['type'] }.transform_values(&:count)
+    aggregates = events.group_by { |e| e["type"] }.transform_values(&:count)
     Analytics.bulk_insert(aggregates)
     messages.each(&:ack)
   end

data/lib/cosmo/api/kv.rb

@@ -48,7 +48,7 @@ module Cosmo
 
       def count
         keys.size
-      rescue NATS::KeyValue::NoKeysFoundError
+      rescue NATS::KeyValue::NoKeysFoundError, NATS::JetStream::Error::NotFound
         0
       end
       alias size count

data/lib/cosmo/api/stream.rb

@@ -10,12 +10,13 @@ module Cosmo
       include Enumerable
 
       def self.all
-        client.list_streams.filter_map { new(_1) }
+        client.list_streams.map { new(_1.dig("config", "name")) }
       end
 
       def self.jobs
-        names = Config[:setup][:jobs].keys - %i[scheduled dead]
-        all.select { names.include?(_1.name.to_sym) }
+        client.list_streams.select { _1.dig("config", "metadata", "_cosmo.type") == "jobs" }
+                           .reject { %w[scheduled dead].include?(_1.dig("config", "name")) }
+                           .map { new(_1.dig("config", "name")) }
       end
 
       def self.client
@@ -41,7 +42,7 @@ module Cosmo
       alias size total
 
       def retries
-        client.list_consumers(name).sum { it["num_redelivered"].to_i }
+        client.list_consumers(name).sum { _1["num_redelivered"].to_i }
       end
 
       def each
@@ -100,6 +101,18 @@ module Cosmo
         client.delete_message(name, seq)
       end
 
+      def pause!
+        client.pause_stream(name)
+      end
+
+      def unpause!
+        client.unpause_stream(name)
+      end
+
+      def paused?
+        client.stream_paused?(name)
+      end
+
       private
 
       def client

data/lib/cosmo/cli.rb

@@ -106,11 +106,12 @@ module Cosmo
             configs.each do |name, config|
               Client.instance.stream_info(name)
             rescue NATS::JetStream::Error::NotFound
-              Client.instance.create_stream(name, config)
+              meta = { metadata: { "_cosmo.type" => "jobs" } }
+              Client.instance.create_stream(name, config.merge(meta))
             end
           end
 
-          puts "Cosmo streams were created/updated"
+          puts "Cosmo streams were created successfully"
           exit(0)
         end
 

data/lib/cosmo/client.rb

@@ -38,18 +38,42 @@ module Cosmo
       js.delete_stream(name, params)
     end
 
+    def update_stream(name, config)
+      js.update_stream(name: name, **config)
+    end
+
     def list_streams
       response = nc.request("$JS.API.STREAM.LIST", "")
       data = Utils::Json.parse(response.data, symbolize_names: false)
       return [] if data.nil? || data["streams"].nil?
 
-      data["streams"].filter_map { _1.dig("config", "name") }
+      data["streams"]
+    end
+
+    def pause_stream(name)
+      config = stream_info(name).config.to_h
+      config[:metadata] ||= {}
+      config[:metadata][:"_cosmo.paused"] = "true"
+      update_stream(name, config)
+    end
+
+    def unpause_stream(name)
+      config = stream_info(name).config.to_h
+      config[:metadata] ||= {}
+      config[:metadata].delete(:"_cosmo.paused")
+      update_stream(name, config)
+    end
+
+    def stream_paused?(name)
+      stream_info(name).config.metadata&.[](:"_cosmo.paused") == "true"
+    rescue NATS::IO::Timeout
+      false
     end
 
     def list_consumers(stream_name)
       response = nc.request("$JS.API.CONSUMER.LIST.#{stream_name}", "")
-      data = Utils::Json.parse(response.data, symbolize_names: false)
-      data["consumers"]
+      data = Utils::Json.parse(response.data, default: {}, symbolize_names: false)
+      Array(data["consumers"])
     end
 
     def consumer_info(stream_name, consumer_name)

data/lib/cosmo/config.rb

@@ -4,7 +4,7 @@ require "yaml"
 require "forwardable"
 
 module Cosmo
-  class Config
+  class Config < ::Hash
     NANO = 1_000_000_000
     DEFAULT_PATH = "config/cosmo.yml"
 
@@ -59,45 +59,18 @@ module Cosmo
       @instance ||= new
     end
 
-    def self.system
-      @system ||= {}
-    end
-
-    def initialize
-      @config = nil
-      @system = {}
-      @defaults = self.class.parse_file(File.expand_path("defaults.yml", __dir__))
-    end
-
-    def [](key)
-      dig(key)
-    end
-
-    def fetch(key, default = nil)
-      return @config.fetch(key, default) if @config && Utils::Hash.keys?(@config, key)
-
-      @defaults.fetch(key, default)
-    end
-
-    def dig(*keys)
-      return @config&.dig(*keys) if @config && Utils::Hash.keys?(@config, *keys)
-
-      @defaults.dig(*keys)
-    end
-
-    def to_h
-      Utils::Hash.merge(@defaults, @config)
+    def self.internal
+      @internal ||= {}
     end
 
     def set(...)
-      @config ||= {}
-      Utils::Hash.set(@config, ...)
+      Utils::Hash.set(self, ...)
     end
 
     def load(path = nil)
       return unless path
 
-      @config = self.class.parse_file(path)
+      replace(self.class.parse_file(path))
     end
   end
 end

data/lib/cosmo/engine.rb

@@ -25,7 +25,7 @@ module Cosmo
 
     def run(type, options)
       handler = Utils::Signal.trap(:INT, :TERM)
-      Logger.info "Starting processing, hit Ctrl-C to stop"
+      Logger.info "Starting processing, hit Ctrl-C to stop [concurrency=#{@concurrency}]"
 
       processor_classes = type && PROCESSORS.key?(type.to_sym) ? [PROCESSORS[type.to_sym]] : PROCESSORS.values
       @processors = processor_classes.map { _1.run(@pool, @running, options) }

data/lib/cosmo/job/processor.rb

@@ -2,74 +2,28 @@
 
 module Cosmo
   module Job
-    class Processor < ::Cosmo::Processor # rubocop:disable Metrics/ClassLength
-      def initialize(pool, running, options)
-        super
-        @weights = []
-      end
-
-      def stop(timeout = Config[:timeout])
-        @running.make_false
-        @pool.shutdown
-        @consumers.each { |(s, _)| s.unsubscribe rescue nil }
-        @pool.wait_for_termination(timeout)
-        [@work_thread, @schedule_thread].compact.each { _1.join(timeout) || _1.kill }
-        @consumers.clear
-      end
-
+    class Processor < ::Cosmo::Processor
       private
 
-      def run_loop
-        @work_thread = Thread.new { work_loop }
-        @schedule_thread = Thread.new { schedule_loop }
-      end
-
       def setup
         jobs_config = Config.dig(:consumers, :jobs)
         jobs_config&.each do |stream_name, config|
-          config = config.dup
-          consumer_name = "consumer-#{stream_name}"
-          subject = config.delete(:subject)
-          priority = config.delete(:priority)
-          @weights += ([stream_name] * priority.to_i) if priority
-          subscription = client.subscribe(subject, consumer_name, config)
-          @consumers << [subscription, stream_name]
-        end
-      end
-
-      def work_loop # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
-        shutdown = false
+          next if stream_name == :scheduled # scheduled jobs are handled in schedule_loop
 
-        while running?
-          break if shutdown
-
-          @weights.shuffle.each do |stream_name|
-            break unless running?
-
-            begin
-              timeout = ENV.fetch("COSMO_JOBS_FETCH_TIMEOUT", 0.1).to_f
-              @pool.post do
-                subscription = @consumers.find { |(_, sn)| sn == stream_name }&.first
-                messages = lock(stream_name) { fetch(subscription, batch_size: 1, timeout:) }
-                process(messages) if messages&.any?
-              end
-            rescue Concurrent::RejectedExecutionError
-              shutdown = true
-              break # pool doesn't accept new jobs, we are shutting down
-            end
-
-            break unless running?
-          end
+          @consumers << subscribe(stream_name, config)
         end
       end
 
       def schedule_loop # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength, Metrics/AbcSize
+        config = Config.dig(:consumers, :jobs, :scheduled)
+        return unless config
+
+        subscription, = subscribe(:scheduled, config)
         while running?
           break unless running?
 
           now = Time.now.to_i
           timeout = ENV.fetch("COSMO_JOBS_SCHEDULER_FETCH_TIMEOUT", 5).to_f
-          subscription = @consumers.find { |(_, sn)| sn == :scheduled }&.first
           messages = fetch(subscription, batch_size: 100, timeout:)
           messages&.each do |message|
             headers = message.header.except("X-Stream", "X-Subject", "X-Execute-At", "Nats-Expected-Stream")
@@ -90,7 +44,7 @@ module Cosmo
         end
       end
 
-      def process(messages) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def process(messages, _) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
         message = messages.first
         Logger.debug "received messages #{messages.inspect}"
         data = Utils::Json.parse(message.data)
@@ -148,6 +102,15 @@ module Cosmo
         true
       end
 
+      def subscribe(stream_name, config)
+        config = config.dup
+        config[:batch_size] = 1
+        config[:stream] = stream_name
+        consumer_name = "consumer-#{stream_name}"
+        subscription = client.subscribe(config[:subject], consumer_name, config.except(:subject, :priority, :stream, :batch_size))
+        [subscription, config, nil]
+      end
+
       def drop_message(message, data)
         message.term
         Logger.debug "job dropped #{data[:jid]}"
@@ -161,16 +124,28 @@ module Cosmo
         Logger.debug "job moved #{data&.dig(:jid)} to DLQ"
       end
 
+      def scheduler?
+        true
+      end
+
+      def consumers
+        @weights ||= @consumers.filter_map { |(_, c, _)| [c[:stream]] * [c[:priority].to_i, 1].max }.flatten
+        @weights.shuffle.map { |s| @consumers.find { |(_, c, _)| c[:stream] == s } }
+      end
+
+      def fetch_subjects(config)
+        config[:subject]
+      end
+
+      def fetch_timeout(_config)
+        ENV.fetch("COSMO_JOBS_FETCH_TIMEOUT", 0.1).to_f
+      end
+
       def with_stats(message, &block)
         API::Busy.instance.with(message) do
           API::Counter.instance.with(&block)
         end
       end
-
-      def lock(stream_name, &)
-        @mutexes ||= Hash.new { |h, k| h[k] = Mutex.new }
-        @mutexes[stream_name].synchronize(&)
-      end
     end
   end
 end

data/lib/cosmo/logger.rb

@@ -60,7 +60,10 @@ module Cosmo
     end
 
     def self.instance
-      @instance ||= ::Logger.new($stdout).tap { _1.formatter = SimpleFormatter.new }
+      @instance ||= ::Logger.new($stdout).tap do |logger|
+        logger.formatter = SimpleFormatter.new
+        logger.level = ::Logger::Severity.coerce(ENV.fetch("COSMO_LOG_LEVEL", "info"))
+      end
     end
 
     def self.instance=(logger)