cosmonats 0.1.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c001e33a420b73f46eebfe6ecffca697ef7d67f0436a2b70a88ed6207b3846a5
-  data.tar.gz: 2fd541b16a24b5a461ea60078be20d92a0c815d0a09996fc9e419f958fce31ea
+  metadata.gz: f850638b66506538b131180c03b64a13e31cd86f924503703812da49b6ba3d20
+  data.tar.gz: 0dfdab08193600f3bca2c32558ffb016a38657f9488789770eea5e60612281e9
 SHA512:
-  metadata.gz: b401aaac7b86f8d08167d256814bed8cd8332aa1f9d82e97548f89d5f3770f95d90a8c36dc711eb0798fbc21ec654d21f629879e3fb56e8d3ecc4c943411be24
-  data.tar.gz: 8ff68d161c0f1f15dabddab8e36ad37363456126c5332bdfe636620e68c91510d690048585fa2064538e77ef3be6c1f892be4f57878de844d9f32d4a46815da3
+  metadata.gz: 56f46ac8a8c1409b1583033ad8b5cf9e513652fae2643b07344aa8f8bf1cca80f948463af92e2a989ec9679cc830ac05a6b873b1f0c9b5e10a1e6ef5e8399ad0
+  data.tar.gz: '086ae5c5b26471417e54c37d2f782c0c103340953e9a27ca8c02131478122f01ede30f2f8ab5a0b29c81ce7b4ea07a5c8de8e7f51957e4d9d5dfd5d670a0ff08'

data/README.md

@@ -95,62 +95,69 @@ gem "cosmonats"
 
 **Requirements:** Ruby 3.1.0+, NATS Server ([installation guide](https://docs.nats.io/running-a-nats-service/introduction/installation))
 
+Add these lines to config/routes.rb:
+```ruby
+require "cosmo/web"
+
+Rails.application.routes.draw do
+  mount Cosmo::Web => "/cosmo" # access web UI at http://localhost:3000/cosmo
+  ...
+end
+```
+
 
 ## 🚀 Quick Start
 
-### 1. Create a Job
+### 1. Create `config/cosmo.yml` and run `bundle exec cosmo -S` to create streams in NATS:
+
+```yaml
+concurrency: 5
+max_retries: 3
+
+consumers:
+  jobs:
+    default:
+      ack_policy: explicit
+      max_deliver: 10
+      max_ack_pending: 10
+      ack_wait: 15
+      subject: jobs.%{name}.>
+
+setup:
+  jobs:
+    default:
+      storage: file
+      retention: workqueue
+      subjects: ["jobs.%{name}.>"]
+      allow_direct: true
+``` 
+
+### 2. Create a Job in app/workers
 
 ```ruby
 class SendEmailJob
   include Cosmo::Job
 
-  # configure job options (optional)
   options stream: :default, retry: 3, dead: true
 
   def perform(user_id, email_type)
-    user = User.find(user_id)
-    UserMailer.send(email_type, user).deliver_now
+    # Pretend to send email to user: UserMailer.send(email_type, user_id).deliver_now
+    sleep 0.5 # Simulate work
+    puts "#{user_id}, #{email_type}"
   end
 end
 ```
 
-### 2. Enqueue Jobs
+### 3. Enqueue Jobs
 
 ```ruby
-SendEmailJob.perform_async(123, 'welcome')           # Immediately
-SendEmailJob.perform_in(1.hour, 123, 'reminder')     # Delayed
-SendEmailJob.perform_at(1.day.from_now, 123, 'test') # Scheduled
-```
-
-### 3. Configure (config/cosmo.yml)
-
-```yaml
-concurrency: 10
-max_retries: 3
-
-consumers:
-  jobs:
-    default:
-      ack_policy: explicit
-      max_deliver: 3
-      max_ack_pending: 3
-      ack_wait: 60
-
-streams:
-  default:
-    storage: file
-    retention: workqueue
-    subjects: ["jobs.default.>"]
+10.times { |i| SendEmailJob.perform_async(i, "welcome") }
 ```
 
-### 4. Setup & Run
+### 4. Run
 
 ```bash
-# Setup streams
-cosmo -C config/cosmo.yml --setup
-
-# Start processing
-cosmo -C config/cosmo.yml -c 10 -r ./app/jobs jobs
+bundle exec cosmo jobs
 ```
 
 
@@ -182,7 +189,7 @@ end
 # Usage
 ReportJob.perform_async(42)                              # Enqueue now
 ReportJob.perform_in(30.minutes, 42)                     # Delayed
-ReportJob.perform_at(Time.parse('2026-01-25 10:00'), 42) # Scheduled
+ReportJob.perform_at(Time.parse("2026-01-25 10:00"), 42) # Scheduled
 ```
 
 ### Streams
@@ -221,8 +228,8 @@ end
 
 # Publishing
 ClicksProcessor.publish(
-  { user_id: 123, page: '/home' },
-  subject: 'events.clicks.homepage'
+  { user_id: 123, page: "/home" },
+  subject: "events.clicks.homepage"
 )
 
 # Message acknowledgment strategies
@@ -235,31 +242,86 @@ message.term                         # Permanent failure, no retry
 
 **File-based (config/cosmo.yml):**
 ```yaml
-timeout: 25     # Shutdown timeout in seconds
-concurrency: 10 # Number of worker threads
-max_retries: 3  # Default max retries
+timeout: 25                 # Shutdown timeout in seconds
+concurrency: &concurrency 1 # Number of worker threads
+max_retries: &max_retries 3 # Default max retries
+
+stream_config: &stream_config
+  storage: file         # storage type (file or memory)
+  retention: workqueue  # retention policy (limits, interest, workqueue)
+  duplicate_window: 120 # time window for duplicate message detection in seconds
+  discard: old          # discard new messages when stream is full (discard new or old)
+  allow_direct: true    # allow direct messages to stream, required for web UI
+  subjects:
+    - jobs.%{name}.>    # subject pattern for stream, %{name} will be replaced with stream name
+
+consumer_config: &consumer_config
+  ack_policy: explicit    # ack policy (explicit, none, all), each individual message must be acknowledged
+  max_deliver: 10         # maximum number of times a message will be delivered before it's considered failed
+  max_ack_pending: 20     # maximum number of messages with pending ack for this consumer
+  ack_wait: 60            # time in seconds to wait for an ack before redelivering the message
+  subject: jobs.%{name}.> # subject pattern for consumer, %{name} will be replaced with stream name
 
 consumers:
-  streams:
-    - class: MyStream
-      batch_size: 50
-      consumer:
-        ack_policy: explicit
-        max_deliver: 3
-        subjects: ["events.>"]
-
-streams:
-  my_stream:
-    storage: file         # or memory
-    retention: workqueue  # or limits
-    max_age: 86400       # 1d in seconds
-    subjects: ["events.>"]
+  jobs:
+    critical:
+      <<: *consumer_config
+      priority: 50
+    high:
+      <<: *consumer_config
+      priority: 30
+    default:
+      <<: *consumer_config
+      priority: 15
+    low:
+      <<: *consumer_config
+      priority: 5
+    scheduled:
+      <<: *consumer_config
+      max_deliver: 1
+      max_ack_pending: 100
+      ack_wait: 10
+
+setup:
+  jobs:
+    critical:
+      <<: *stream_config
+      description: Very critical priority jobs
+    high:
+      <<: *stream_config
+      description: Higher priority jobs
+    default:
+      <<: *stream_config
+      description: Default priority jobs
+    low:
+      <<: *stream_config
+      description: Lower priority jobs
+    scheduled:
+      <<: *stream_config
+      description: Scheduled jobs
+    dead:
+      <<: *stream_config
+      retention: limits
+      max_msgs: 10000
+      max_age: 604800 # 7d
+      description: Broken jobs (DLQ)
+
+development:
+  verbose: false
+  concurrency: *concurrency
+
+staging:
+  verbose: true
+  concurrency: 3
+
+production:
+  concurrency: 3
 ```
 
 **Programmatic:**
 ```ruby
 Cosmo::Config.set(:concurrency, 20)
-Cosmo::Config.set(:streams, :custom, { storage: 'file', subjects: ['custom.>'] })
+Cosmo::Config.set(:setup, :streams, :custom, { storage: "file", subjects: ["custom.>"] })
 ```
 
 **Environment variables:**
@@ -327,10 +389,10 @@ end
 **Testing:**
 ```ruby
 # Synchronous execution
-SendEmailJob.perform_sync(123, 'test')
+SendEmailJob.perform_sync(123, "test")
 
 # Test job creation
-jid = SendEmailJob.perform_async(123, 'welcome')
+jid = SendEmailJob.perform_async(123, "welcome")
 assert_kind_of String, jid
 ```
 
@@ -437,7 +499,7 @@ sudo systemctl status cosmo
 **Stream Metrics:**
 ```ruby
 client = Cosmo::Client.instance
-info = client.stream_info('default')
+info = client.stream_info("default")
 
 info.state.messages       # Total messages
 info.state.bytes          # Total bytes
@@ -464,8 +526,8 @@ class EmailJob
   end
 end
 
-EmailJob.perform_async(123, 'welcome')
-EmailJob.perform_in(1.day, 123, 'followup')
+EmailJob.perform_async(123, "welcome")
+EmailJob.perform_in(1.day, 123, "followup")
 ```
 
 **Image Processing Pipeline:**
@@ -474,12 +536,12 @@ class ImageProcessor
   include Cosmo::Stream
   options(
     stream: :images,
-    consumer: { subjects: ['images.uploaded.>'] }
+    consumer: { subjects: ["images.uploaded.>"] }
   )
 
   def process_one
-    processed = ImageService.process(message.data['url'])
-    publish(processed, subject: 'images.processed.optimized')
+    processed = ImageService.process(message.data["url"])
+    publish(processed, subject: "images.processed.optimized")
     message.ack
   rescue => e
     logger.error "Processing failed: #{e.message}"
@@ -487,18 +549,18 @@ class ImageProcessor
   end
 end
 
-ImageProcessor.publish({ url: 'https://example.com/image.jpg' }, subject: 'images.uploaded.user')
+ImageProcessor.publish({ url: "https://example.com/image.jpg" }, subject: "images.uploaded.user")
 ```
 
 **Real-Time Analytics:**
 ```ruby
 class AnalyticsAggregator
   include Cosmo::Stream
-  options batch_size: 1000, consumer: { subjects: ['events.*.>'] }
+  options batch_size: 1000, consumer: { subjects: ["events.*.>"] }
 
   def process(messages)
     events = messages.map(&:data)
-    aggregates = events.group_by { |e| e['type'] }.transform_values(&:count)
+    aggregates = events.group_by { |e| e["type"] }.transform_values(&:count)
     Analytics.bulk_insert(aggregates)
     messages.each(&:ack)
   end

data/lib/cosmo/api/busy.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module Cosmo
+  module API
+    class Busy
+      TTL = 70
+      HEARTBEAT = 30
+      BUCKET = "cosmostats"
+
+      def self.instance
+        @instance ||= new
+      end
+
+      def initialize
+        @messages = {}
+        @kv = KV.new(BUCKET, { ttl: TTL })
+      end
+
+      def with(message)
+        add(message)
+        yield
+      ensure
+        delete(message)
+      end
+
+      def add(message)
+        @thread ||= Thread.new { heartbeat_loop }
+        seq = message.metadata.sequence.stream
+        value = Utils::Json.dump({ data: message.data, stream: message.metadata.stream, worker: worker_id, started_at: Time.now.to_i })
+        @messages[seq] = value
+        @kv.set(seq, value)
+      end
+
+      def delete(message)
+        seq = message.metadata.sequence.stream
+        @messages.delete(seq)
+        @kv.purge(seq)
+      end
+
+      def list(limit: 25)
+        @kv.keys(limit:).filter_map { Utils::Json.parse(@kv.get(_1)) }.map { _1.merge(data: Utils::Json.parse(_1[:data])) }
+      end
+
+      def size
+        @kv.size
+      end
+
+      private
+
+      def heartbeat_loop
+        loop do
+          sleep(HEARTBEAT)
+          @messages.dup.each { |seq, value| @kv.set(seq, value) rescue StandardError }
+        rescue StandardError => e
+          Logger.debug "Busy heartbeat error: #{e.class} #{e.message}"
+        end
+      end
+
+      def worker_id
+        @worker_id ||= "#{Socket.gethostname}-#{Process.pid}"
+      end
+    end
+  end
+end

data/lib/cosmo/api/counter.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    class Counter
+      STREAM_NAME = "cosmostats"
+
+      def self.instance
+        @instance ||= new("jobs")
+      end
+
+      def initialize(namespace)
+        @namespace = namespace
+      end
+
+      def with
+        result = yield
+        increment(:processed) if result == true
+        increment(:failed) if result == false
+      rescue Exception # rubocop:disable Lint/RescueException
+        increment(:failed)
+      end
+
+      def increment(key, by: 1)
+        publish(key, "+#{by}")
+      end
+      alias incr increment
+
+      def decrement(key, by: 1)
+        publish(key, "-#{by}")
+      end
+      alias decr decrement
+
+      def reset(key)
+        client.purge(STREAM_NAME, subject(key))
+      end
+
+      def get(key)
+        raw = client.get_message(STREAM_NAME, direct: true, subject: subject(key))
+        Utils::Json.parse(raw.data, default: { "val" => 0 })[:val].to_i
+      rescue NATS::JetStream::Error::NotFound, NATS::JetStream::Error::ServiceUnavailable
+        0
+      end
+
+      private
+
+      def publish(key, value)
+        rescued = nil
+
+        begin
+          client.publish(subject(key), "", header: { "Nats-Incr" => value }).val.to_i
+        rescue NATS::JetStream::Error::NoStreamResponse
+          raise if rescued
+
+          rescued = true
+          client.create_stream(STREAM_NAME, subjects: ["#{STREAM_NAME}.>"], allow_msg_counter: true, allow_direct: true, description: "Cosmo statistics")
+          retry
+        end
+      end
+
+      def subject(key)
+        "#{STREAM_NAME}.#{@namespace}.#{key}"
+      end
+
+      def client
+        @client ||= Client.instance
+      end
+    end
+  end
+end

data/lib/cosmo/api/job.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    class Job
+      attr_reader :message, :stream
+
+      def initialize(stream, message)
+        @stream = stream
+        @message = message
+      end
+
+      def data
+        @data ||= Utils::Json.parse(@message.data)
+      end
+
+      def seq
+        @message.seq
+      end
+
+      def headers
+        @message.headers
+      end
+
+      def execute_at
+        headers&.dig("X-Execute-At")&.to_i
+      end
+
+      def x_stream
+        headers&.dig("X-Stream")
+      end
+
+      def x_subject
+        headers&.dig("X-Subject")
+      end
+
+      def subject
+        @message.subject
+      end
+
+      def timestamp
+        headers&.dig("Nats-Time-Stamp")
+      end
+    end
+  end
+end

data/lib/cosmo/api/kv.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    class KV
+      def initialize(name, options = nil)
+        @name = name
+        @options = Hash(options)
+      end
+
+      def set(key, value)
+        kv.put(key, value.to_s)
+      end
+
+      def get(key)
+        kv.get(key).value
+      rescue NATS::KeyValue::KeyNotFoundError
+        # nop
+      end
+
+      def delete(key)
+        kv.delete(key)
+      end
+
+      def keys(subject = nil, limit: 25)
+        results = []
+        params = { ignore_deletes: true, meta_only: true }
+        watcher = kv.watch(subject || ">", params)
+
+        watcher.each do |entry|
+          break unless entry
+
+          results << entry.key
+          break if results.size >= limit
+        end
+        watcher.stop
+
+        results
+      end
+
+      def purge(key)
+        kv.purge(key)
+      end
+
+      def clean
+        Client.instance.purge("KV_#{@name}", ">")
+      end
+
+      def count
+        keys.size
+      rescue NATS::KeyValue::NoKeysFoundError, NATS::JetStream::Error::NotFound
+        0
+      end
+      alias size count
+
+      private
+
+      def kv
+        @kv ||= Client.instance.kv(@name, **@options)
+      end
+    end
+  end
+end

data/lib/cosmo/api/stats.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "cosmo/api/counter"
+require "cosmo/api/busy"
+
+module Cosmo
+  module API
+    module Stats
+      module_function
+
+      def summary
+        { processed:, failed:, busy:, enqueued:, retries:, scheduled:, dead: }
+      end
+
+      def processed
+        Counter.instance.get(:processed)
+      end
+
+      def failed
+        Counter.instance.get(:failed)
+      end
+
+      def busy
+        Busy.instance.size
+      end
+
+      def enqueued
+        Stream.jobs.sum(&:size)
+      end
+
+      def retries
+        Stream.jobs.sum(&:retries)
+      end
+
+      def scheduled
+        Stream.new("scheduled").size
+      end
+
+      def dead
+        Stream.new("dead").size
+      end
+    end
+  end
+end