cosmonats 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c5a169718d8f3ddb85e2fcc95a5e1b6c0fd3061c1ad4410bd7bad85add934c6
-  data.tar.gz: aa9715564bdbead26f876ff9de8c9119084015fb3e0d52e2ec6b126c3d3ad0a0
+  metadata.gz: b63356c69b61ea32b4791b519830003e4f42330d39947a332cfbfdb20ed91c7d
+  data.tar.gz: 6cc0401b58b06038dcc4dc7e4bea9ff695662fa1b251b89fd681a66e893387c7
 SHA512:
-  metadata.gz: 41bf7cc2664b566e0f70f9b4045a921abe7afc0e0cd33897f46dd975d4148aead93eaf7faeaffd7364268acee1713b08daeb151c7b124412de2983970787f97b
-  data.tar.gz: 761a89e2379601c71b75e6c13f63e0c7c0dda9fa7f5631aa6b0f8bab7493f3fe29bee1e562677be8f22ac45b1597f4a0f34660b91e999a7eceb7367d0db50d33
+  metadata.gz: 5c7ef63abf649154cb9a5e5662e29297cd4b171810f819b16401f08651a5ee3a8c3111421ed3873ea4c12477d05cd6329e18f67feda91edadf7605eadf2a4b91
+  data.tar.gz: d1bb560e31a2c0fb6c321d52599a543a372f184e136f29778a44451648eaa7beed1405d83d1fe996f87b8ee029432d8f395c3fad348a7eaacb73de958ad31573

data/README.md

@@ -248,18 +248,19 @@ consumers:
         max_deliver: 3
         subjects: ["events.>"]
 
-streams:
-  my_stream:
-    storage: file         # or memory
-    retention: workqueue  # or limits
-    max_age: 86400       # 1d in seconds
-    subjects: ["events.>"]
+setup:
+  streams:
+    my_stream:
+      storage: file         # or memory
+      retention: workqueue  # or limits
+      max_age: 86400       # 1d in seconds
+      subjects: ["events.>"]
 ```
 
 **Programmatic:**
 ```ruby
 Cosmo::Config.set(:concurrency, 20)
-Cosmo::Config.set(:streams, :custom, { storage: 'file', subjects: ['custom.>'] })
+Cosmo::Config.set(:setup, :streams, :custom, { storage: 'file', subjects: ['custom.>'] })
 ```
 
 **Environment variables:**

data/lib/cosmo/api/busy.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module Cosmo
+  module API
+    class Busy
+      TTL = 70
+      HEARTBEAT = 30
+      BUCKET = "cosmostats"
+
+      def self.instance
+        @instance ||= new
+      end
+
+      def initialize
+        @messages = {}
+        @kv = KV.new(BUCKET, { ttl: TTL })
+      end
+
+      def with(message)
+        add(message)
+        yield
+      ensure
+        delete(message)
+      end
+
+      def add(message)
+        @thread ||= Thread.new { heartbeat_loop }
+        seq = message.metadata.sequence.stream
+        value = Utils::Json.dump({ data: message.data, stream: message.metadata.stream, worker: worker_id, started_at: Time.now.to_i })
+        @messages[seq] = value
+        @kv.set(seq, value)
+      end
+
+      def delete(message)
+        seq = message.metadata.sequence.stream
+        @messages.delete(seq)
+        @kv.purge(seq)
+      end
+
+      def list(limit: 25)
+        @kv.keys(limit:).filter_map { Utils::Json.parse(@kv.get(_1)) }.map { _1.merge(data: Utils::Json.parse(_1[:data])) }
+      end
+
+      def size
+        @kv.size
+      end
+
+      private
+
+      def heartbeat_loop
+        loop do
+          sleep(HEARTBEAT)
+          @messages.dup.each { |seq, value| @kv.set(seq, value) rescue StandardError }
+        rescue StandardError => e
+          Logger.debug "Busy heartbeat error: #{e.class} #{e.message}"
+        end
+      end
+
+      def worker_id
+        @worker_id ||= "#{Socket.gethostname}-#{Process.pid}"
+      end
+    end
+  end
+end

data/lib/cosmo/api/counter.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    class Counter
+      STREAM_NAME = "cosmostats"
+
+      def self.instance
+        @instance ||= new("jobs")
+      end
+
+      def initialize(namespace)
+        @namespace = namespace
+      end
+
+      def with
+        result = yield
+        increment(:processed) if result == true
+        increment(:failed) if result == false
+      rescue Exception # rubocop:disable Lint/RescueException
+        increment(:failed)
+      end
+
+      def increment(key, by: 1)
+        publish(key, "+#{by}")
+      end
+      alias incr increment
+
+      def decrement(key, by: 1)
+        publish(key, "-#{by}")
+      end
+      alias decr decrement
+
+      def reset(key)
+        client.purge(STREAM_NAME, subject(key))
+      end
+
+      def get(key)
+        raw = client.get_message(STREAM_NAME, direct: true, subject: subject(key))
+        Utils::Json.parse(raw.data, default: { "val" => 0 })[:val].to_i
+      rescue NATS::JetStream::Error::NotFound, NATS::JetStream::Error::ServiceUnavailable
+        0
+      end
+
+      private
+
+      def publish(key, value)
+        rescued = nil
+
+        begin
+          client.publish(subject(key), "", header: { "Nats-Incr" => value }).val.to_i
+        rescue NATS::JetStream::Error::NoStreamResponse
+          raise if rescued
+
+          rescued = true
+          client.create_stream(STREAM_NAME, subjects: ["#{STREAM_NAME}.>"], allow_msg_counter: true, allow_direct: true, description: "Cosmo statistics")
+          retry
+        end
+      end
+
+      def subject(key)
+        "#{STREAM_NAME}.#{@namespace}.#{key}"
+      end
+
+      def client
+        @client ||= Client.instance
+      end
+    end
+  end
+end

data/lib/cosmo/api/job.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    class Job
+      attr_reader :message, :stream
+
+      def initialize(stream, message)
+        @stream = stream
+        @message = message
+      end
+
+      def data
+        @data ||= Utils::Json.parse(@message.data)
+      end
+
+      def seq
+        @message.seq
+      end
+
+      def headers
+        @message.headers
+      end
+
+      def execute_at
+        headers&.dig("X-Execute-At")&.to_i
+      end
+
+      def x_stream
+        headers&.dig("X-Stream")
+      end
+
+      def x_subject
+        headers&.dig("X-Subject")
+      end
+
+      def subject
+        @message.subject
+      end
+
+      def timestamp
+        headers&.dig("Nats-Time-Stamp")
+      end
+    end
+  end
+end

data/lib/cosmo/api/kv.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module API
+    class KV
+      def initialize(name, options = nil)
+        @name = name
+        @options = Hash(options)
+      end
+
+      def set(key, value)
+        kv.put(key, value.to_s)
+      end
+
+      def get(key)
+        kv.get(key).value
+      rescue NATS::KeyValue::KeyNotFoundError
+        # nop
+      end
+
+      def delete(key)
+        kv.delete(key)
+      end
+
+      def keys(subject = nil, limit: 25)
+        results = []
+        params = { ignore_deletes: true, meta_only: true }
+        watcher = kv.watch(subject || ">", params)
+
+        watcher.each do |entry|
+          break unless entry
+
+          results << entry.key
+          break if results.size >= limit
+        end
+        watcher.stop
+
+        results
+      end
+
+      def purge(key)
+        kv.purge(key)
+      end
+
+      def clean
+        Client.instance.purge("KV_#{@name}", ">")
+      end
+
+      def count
+        keys.size
+      rescue NATS::KeyValue::NoKeysFoundError
+        0
+      end
+      alias size count
+
+      private
+
+      def kv
+        @kv ||= Client.instance.kv(@name, **@options)
+      end
+    end
+  end
+end

data/lib/cosmo/api/stats.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "cosmo/api/counter"
+require "cosmo/api/busy"
+
+module Cosmo
+  module API
+    module Stats
+      module_function
+
+      def summary
+        { processed:, failed:, busy:, enqueued:, retries:, scheduled:, dead: }
+      end
+
+      def processed
+        Counter.instance.get(:processed)
+      end
+
+      def failed
+        Counter.instance.get(:failed)
+      end
+
+      def busy
+        Busy.instance.size
+      end
+
+      def enqueued
+        Stream.jobs.sum(&:size)
+      end
+
+      def retries
+        Stream.jobs.sum(&:retries)
+      end
+
+      def scheduled
+        Stream.new("scheduled").size
+      end
+
+      def dead
+        Stream.new("dead").size
+      end
+    end
+  end
+end

data/lib/cosmo/api/stream.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "cosmo/api/job"
+
+module Cosmo
+  module API
+    class Stream
+      LIMIT = 20
+
+      include Enumerable
+
+      def self.all
+        client.list_streams.filter_map { new(_1) }
+      end
+
+      def self.jobs
+        names = Config[:setup][:jobs].keys - %i[scheduled dead]
+        all.select { names.include?(_1.name.to_sym) }
+      end
+
+      def self.client
+        @client ||= Client.instance
+      end
+
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+      end
+
+      def info
+        info = client.stream_info(name)
+        { state: info.state, config: info.config }
+      end
+
+      def total
+        info[:state].messages.to_i
+      rescue StandardError
+        0
+      end
+      alias size total
+
+      def retries
+        client.list_consumers(name).sum { it["num_redelivered"].to_i }
+      end
+
+      def each
+        return if total.zero?
+
+        state = info[:state]
+        current = @offset || state.first_seq.to_i
+        last = state.last_seq.to_i
+
+        loop do
+          break if current > last
+
+          job = message(current)
+          break unless job
+
+          yield job
+          current += 1
+        end
+      end
+
+      def offset(value)
+        @offset = value.to_i
+        self
+      end
+
+      def messages(page: nil, limit: nil)
+        jobs = []
+        limit = (limit || LIMIT).to_i
+        state = info[:state]
+        start = state.first_seq.to_i
+        start += (page.to_i - 1) * limit if page
+
+        offset(start).each do |message|
+          jobs << message
+          break if jobs.size >= limit
+        end
+
+        jobs
+      end
+
+      def message(seq)
+        Job.new(name, client.get_message(name, seq: seq, direct: true))
+      rescue NATS::JetStream::Error::NotFound
+        # nop, acked/nacked
+      end
+
+      def retry(seq)
+        job = message(seq)
+        return unless job
+
+        client.publish(job.x_subject, job.message.data)
+        delete(seq)
+      end
+
+      def delete(seq)
+        client.delete_message(name, seq)
+      end
+
+      private
+
+      def client
+        self.class.client
+      end
+    end
+  end
+end

data/lib/cosmo/api.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "cosmo/api/stream"
+require "cosmo/api/counter"
+require "cosmo/api/kv"
+require "cosmo/api/stats"
+
+module Cosmo
+  module API
+  end
+end

data/lib/cosmo/cli.rb

@@ -102,10 +102,12 @@ module Cosmo
           load_config(flags)
           boot_application
 
-          Config[:streams].each do |name, config|
-            Client.instance.stream_info(name)
-          rescue NATS::JetStream::Error::NotFound
-            Client.instance.create_stream(name, config)
+          Config[:setup]&.each_value do |configs|
+            configs.each do |name, config|
+              Client.instance.stream_info(name)
+            rescue NATS::JetStream::Error::NotFound
+              Client.instance.create_stream(name, config)
+            end
           end
 
           puts "Cosmo streams were created/updated"

data/lib/cosmo/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "nats/client"
+require "cosmo/utils/overrides"
 
 module Cosmo
   class Client
@@ -11,7 +12,9 @@ module Cosmo
     attr_reader :nc, :js
 
     def initialize(nats_url: ENV.fetch("NATS_URL", "nats://localhost:4222"))
+      Logger.debug "Connecting to NATS server at #{nats_url}..."
       @nc = NATS.connect(nats_url)
+      Logger.debug "Connection established"
       @js = @nc.jetstream
     end
 
@@ -43,8 +46,38 @@ module Cosmo
       data["streams"].filter_map { _1.dig("config", "name") }
     end
 
-    def get_message(name, seq)
-      js.get_msg(name, seq: seq)
+    def list_consumers(stream_name)
+      response = nc.request("$JS.API.CONSUMER.LIST.#{stream_name}", "")
+      data = Utils::Json.parse(response.data, symbolize_names: false)
+      data["consumers"]
+    end
+
+    def consumer_info(stream_name, consumer_name)
+      js.consumer_info(stream_name, consumer_name)
+    end
+
+    def get_message(name, **options)
+      js.get_msg(name, **options)
+    end
+
+    def delete_message(name, seq)
+      response = nc.request("$JS.API.STREAM.MSG.DELETE.#{name}", JSON.dump({ seq: seq }))
+      Utils::Json.parse(response.data, symbolize_names: false)
+    end
+
+    def purge(stream_name, subject)
+      payload = subject ? Utils::Json.dump({ filter: subject }) : ""
+      response = @nc.request("$JS.API.STREAM.PURGE.#{stream_name}", payload)
+      result = Utils::Json.parse(response.data, default: {}, symbolize_names: false)
+      raise NATS::JetStream::Error, result.dig("error", "description") if result["error"]
+
+      result["purged"] # number of messages purged
+    end
+
+    def kv(name, **options)
+      js.key_value(name)
+    rescue NATS::KeyValue::BucketNotFoundError
+      js.create_key_value({ bucket: name }.merge(options))
     end
 
     def close

data/lib/cosmo/config.rb

@@ -18,7 +18,7 @@ module Cosmo
       YAML.load_file(path, aliases: true).tap { normalize!(_1) }
     end
 
-    def self.normalize!(config) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def self.normalize!(config) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
       Utils::Hash.symbolize_keys!(config)
 
       config[:consumers]&.each_key do |name|
@@ -30,11 +30,13 @@ module Cosmo
         end
       end
 
-      config[:streams]&.each_key do |name|
-        c = config[:streams][name]
-        c[:max_age] = c[:max_age].to_i * NANO if c[:max_age]
-        c[:duplicate_window] = c[:duplicate_window].to_i * NANO if c[:duplicate_window]
-        c[:subjects] = c[:subjects].map { |s| format(s, name: name) } if c[:subjects]
+      config[:setup]&.each_key do |type|
+        config[:setup][type]&.each_key do |name|
+          c = config[:setup][type][name]
+          c[:max_age] = c[:max_age].to_i * NANO if c[:max_age]
+          c[:duplicate_window] = c[:duplicate_window].to_i * NANO if c[:duplicate_window]
+          c[:subjects] = c[:subjects].map { |s| format(s, name: name) } if c[:subjects]
+        end
       end
     end
 

data/lib/cosmo/defaults.yml

@@ -27,39 +27,40 @@ consumers:
       max_ack_pending: 100
       ack_wait: 10
 
-streams:
-  critical:
-    <<: &config
-      storage: file
-      retention: workqueue
-      duplicate_window: 120 # 2m
-      discard: old
-      allow_direct: true
-      subjects:
-        - jobs.%{name}.>
-    description: Very critical priority jobs
-  high:
-    <<: *config
-    description: Higher priority jobs
-  default:
-    <<: *config
-    description: Default priority jobs
-  low:
-    <<: *config
-    description: Lower priority jobs
-  scheduled:
-    <<: *config
-    description: Scheduled jobs
-  dead:
-    <<: *config
-    retention: limits
-    max_msgs: 10000
-    max_age: 604800 # 7d
-    description: Broken jobs (DLQ)
+setup:
+  jobs:
+    critical:
+      <<: &config
+        storage: file
+        retention: workqueue
+        duplicate_window: 120 # 2m
+        discard: old
+        allow_direct: true
+        subjects:
+          - jobs.%{name}.>
+      description: Very critical priority jobs
+    high:
+      <<: *config
+      description: Higher priority jobs
+    default:
+      <<: *config
+      description: Default priority jobs
+    low:
+      <<: *config
+      description: Lower priority jobs
+    scheduled:
+      <<: *config
+      description: Scheduled jobs
+    dead:
+      <<: *config
+      retention: limits
+      max_msgs: 10000
+      max_age: 604800 # 7d
+      description: Broken jobs (DLQ)
 
 development:
   verbose: false
-  concurrency: 1
+  concurrency: *concurrency
 
 staging:
   verbose: true