cosmonats 0.1.0

data/lib/cosmo/job.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "cosmo/job/data"
+require "cosmo/job/processor"
+
+module Cosmo
+  module Job
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def options(stream: nil, retry: nil, dead: nil)
+        default_options.merge!({ stream:, retry:, dead: }.compact)
+      end
+
+      def perform(*args, async: true, **options)
+        data = Data.new(name, args, default_options.merge(options))
+        unless async
+          payload = Utils::Json.parse(data.to_args[1])
+          raise ArgumentError, "Cannot parse payload" unless payload
+
+          new.perform(*payload[:args])
+          return
+        end
+
+        Publisher.publish_job(data)
+      end
+
+      def perform_async(*args)
+        perform(*args)
+      end
+
+      def perform_at(timestamp, *args)
+        perform(*args, at: timestamp)
+      end
+
+      def perform_in(interval, *args)
+        perform(*args, in: interval)
+      end
+
+      def perform_sync(*args)
+        perform(*args, async: false)
+      end
+
+      def default_options
+        @default_options ||= (superclass.respond_to?(:default_options) ? superclass.default_options : Data::DEFAULTS).dup
+      end
+
+      private
+
+      def client
+        @client ||= Client.instance
+      end
+    end
+
+    attr_reader :jid
+
+    def perform(...)
+      raise NotImplementedError, "#{self.class}#perform must be implemented"
+    end
+
+    def logger
+      Logger.instance
+    end
+  end
+end

data/lib/cosmo/logger.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "logger"
+require "forwardable"
+
+module Cosmo
+  module Logger
+    module Context
+      KEY = :cosmo_context
+
+      def self.with(**options)
+        prev = current
+        Thread.current[KEY] = prev.merge(options)
+        yield if block_given?
+      ensure
+        Thread.current[KEY] = prev if block_given?
+      end
+
+      def self.without(*keys)
+        Thread.current[KEY] = current.except(*keys)
+        nil
+      end
+
+      def self.current
+        Thread.current[KEY] ||= {}
+        Thread.current[KEY]
+      end
+    end
+
+    class BaseFormatter < ::Logger::Formatter
+      def tid
+        (Thread.current.object_id ^ pid).to_s(36)
+      end
+
+      def pid
+        ::Process.pid
+      end
+    end
+
+    class SimpleFormatter < BaseFormatter
+      def call(severity, time, _, msg)
+        options = Context.current.compact.map { |k, v| "#{k}=#{v}" }.join(" ")
+        options &&= " #{options}" unless options.empty?
+        "#{time.utc.iso8601(3)} #{severity} pid=#{pid} tid=#{tid}#{options}: #{msg2str(msg)}\n"
+      end
+    end
+
+    class << self
+      extend Forwardable
+
+      delegate %i[info error debug warn fatal] => :instance
+    end
+
+    def self.with(...)
+      Context.with(...)
+    end
+
+    def self.without(...)
+      Context.without(...)
+    end
+
+    def self.instance
+      @instance ||= ::Logger.new($stdout).tap { _1.formatter = SimpleFormatter.new }
+    end
+  end
+end

data/lib/cosmo/processor.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Cosmo
+  class Processor
+    def self.run(...)
+      new(...).tap(&:run)
+    end
+
+    def initialize(pool, running)
+      @pool = pool
+      @running = running
+      @consumers = {}
+    end
+
+    def run
+      setup
+      return unless @consumers.any?
+
+      @running.make_true
+      run_loop
+    end
+
+    private
+
+    def run_loop
+      raise NotImplementedError
+    end
+
+    def setup
+      raise NotImplementedError
+    end
+
+    def process(...)
+      raise NotImplementedError
+    end
+
+    def running?
+      @running.true?
+    end
+
+    def fetch_messages(stream_name, batch_size:, timeout:)
+      messages = @consumers[stream_name].fetch(batch_size, timeout:)
+      block_given? ? yield(messages) : process(stream_name, messages)
+    rescue NATS::Timeout
+      # No messages, continue
+    end
+
+    def client
+      Client.instance
+    end
+
+    def stopwatch
+      Utils::Stopwatch.new
+    end
+  end
+end

data/lib/cosmo/publisher.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Cosmo
+  class Publisher
+    class << self
+      extend Forwardable
+
+      delegate %i[publish publish_job publish_batch] => :instance
+    end
+
+    def self.instance
+      @instance ||= new
+    end
+
+    def initialize
+      @client = Client.instance
+    end
+
+    def publish(subject, data, serializer: nil, **options)
+      payload = (serializer || Stream::Serializer).serialize(data)
+      @client.publish(subject, payload, **options)
+    end
+
+    def publish_job(data)
+      subject, payload, params = data.to_args
+      @client.publish(subject, payload, **params)
+      data.jid
+    rescue NATS::JetStream::Error::NoStreamResponse
+      raise StreamNotFoundError, params[:stream].to_s
+    end
+
+    def publish_batch(subject, batch, **options)
+      batch.each { publish(subject, _1, **options) }
+    end
+  end
+end

data/lib/cosmo/stream/data.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Cosmo
+  module Stream
+    class Data
+      DEFAULTS = {
+        batch_size: 100,
+        consumer: {
+          ack_policy: "explicit",
+          max_deliver: 1,
+          max_ack_pending: 3,
+          ack_wait: 30,
+          subjects: ["%{name}.>"]
+        },
+        publisher: { subject: "%{name}.default", serializer: nil }
+      }.freeze
+    end
+  end
+end

data/lib/cosmo/stream/message.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Cosmo
+  module Stream
+    class Message
+      extend Forwardable
+
+      delegate %i[subject reply header metadata ack nack term in_progress] => :@msg
+      delegate %i[timestamp num_delivered num_pending] => :metadata
+
+      def initialize(msg, serializer: nil)
+        @msg = msg
+        @serializer = serializer || Serializer
+      end
+
+      def data
+        @serializer.deserialize(@msg.data)
+      end
+
+      def stream_sequence
+        metadata.sequence.stream
+      end
+
+      def consumer_sequence
+        metadata.sequence.consumer
+      end
+    end
+  end
+end

data/lib/cosmo/stream/processor.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Stream
+    class Processor < ::Cosmo::Processor
+      def initialize(pool, running)
+        super
+        @configs = {}
+        @processors = {}
+      end
+
+      private
+
+      def run_loop
+        Thread.new { work_loop }
+      end
+
+      def setup
+        setup_configs
+        setup_processors
+        setup_consumers
+      end
+
+      def work_loop
+        while running?
+          @consumers.each_key do |stream_name|
+            break unless running?
+
+            begin
+              batch_size = @configs[stream_name][:batch_size]
+              timeout = ENV.fetch("COSMO_STREAMS_FETCH_TIMEOUT", 0.1).to_f
+              @pool.post { fetch_messages(stream_name, batch_size:, timeout:) }
+            rescue Concurrent::RejectedExecutionError
+              break # pool doesn't accept new jobs, we are shutting down
+            end
+
+            break unless running?
+          end
+        end
+      end
+
+      def process(stream_name, messages) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        metadata = messages.last.metadata
+        processor = @processors[stream_name]
+        serializer = processor.class.default_options.dig(:publisher, :serializer)
+        messages = messages.map { Message.new(_1, serializer:) }
+
+        Logger.with(
+          seq_stream: metadata.sequence.stream,
+          seq_consumer: metadata.sequence.consumer,
+          num_pending: metadata.num_pending,
+          timestamp: metadata.timestamp
+        ) { Logger.info "start" }
+
+        sw = stopwatch
+        processor.process(messages)
+        Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "done" }
+      rescue StandardError => e
+        Logger.debug e
+        Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
+      rescue Exception # rubocop:disable Lint/RescueException
+        Logger.with(elapsed: sw.elapsed_seconds) { Logger.info "fail" }
+        raise
+      end
+
+      def setup_configs # rubocop:disable Metrics/AbcSize
+        @configs.merge!(
+          Config.dig(:consumers, :streams).to_h do |config|
+            klass = Utils::String.safe_constantize(config[:class])
+            [config[:stream].to_sym, klass ? config.merge(class: klass) : nil]
+          end.compact
+        )
+        @configs.merge!(
+          Config.system[:streams].to_h do |klass|
+            [klass.default_options[:stream].to_sym, klass.default_options.merge(class: klass)]
+          end
+        )
+      end
+
+      def setup_processors
+        @configs.each { |s, c| @processors[s] = c[:class].new }
+      end
+
+      def setup_consumers
+        @configs.each do |stream_name, config|
+          subjects = config.dig(:consumer, :subjects)
+          deliver_policy = Config.deliver_policy(config[:start_position])
+          config, consumer_name = config.values_at(:consumer, :consumer_name)
+          @consumers[stream_name] = client.subscribe(subjects, consumer_name, config.merge(deliver_policy))
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/stream/serializer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Cosmo
+  module Stream
+    module Serializer
+      module_function
+
+      def serialize(data)
+        Utils::Json.dump(data)
+      end
+
+      def deserialize(payload)
+        Utils::Json.parse(payload, symbolize_names: false)
+      end
+    end
+  end
+end

data/lib/cosmo/stream.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "cosmo/stream/data"
+require "cosmo/stream/message"
+require "cosmo/stream/processor"
+require "cosmo/stream/serializer"
+
+module Cosmo
+  module Stream
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.register
+    end
+
+    module ClassMethods
+      def options(stream: nil, consumer_name: nil, batch_size: nil, start_position: nil, consumer: nil, publisher: nil) # rubocop:disable Metrics/ParameterLists
+        default_options.merge!({ stream: stream, consumer_name: consumer_name, batch_size:, start_position:, consumer:, publisher: }.compact)
+      end
+
+      def publish(data, subject: nil, **options)
+        stream = default_options[:stream]
+        subject ||= default_options.dig(:publisher, :subject)
+        Publisher.publish(subject, data, stream: stream, serializer: default_options.dig(:publisher, :serializer), **options)
+      end
+
+      def default_options
+        @default_options ||= Utils::Hash.dup(superclass.respond_to?(:default_options) ? superclass.default_options : Data::DEFAULTS)
+      end
+
+      def register # rubocop:disable Metrics/AbcSize
+        Config.system[:streams] ||= []
+        Config.system[:streams] << self
+
+        # settings are inherited, don't try to modify them
+        return if default_options != Data::DEFAULTS
+
+        class_name = Utils::String.underscore(name)
+        default_options.merge!(stream: class_name,
+                               consumer_name: "consumer-#{class_name}",
+                               publisher: { subject: "#{class_name}.default" })
+        subjects = default_options.dig(:consumer, :subjects)
+        subjects&.map! { format(_1, name: class_name) }
+
+        subject = default_options[:publisher][:subject]
+        default_options[:publisher][:subject] = format(subject, name: class_name)
+      end
+    end
+
+    def process(messages)
+      messages.each do |message|
+        Thread.current[:cosmo_message] = message
+        process_one
+      ensure
+        Thread.current[:cosmo_message] = nil
+      end
+    end
+    alias process_many process
+    alias process_batch process
+
+    def process_one
+      raise NotImplementedError, "#{self.class}#process_one must be implemented"
+    end
+
+    def logger
+      Logger.instance
+    end
+
+    def message
+      Thread.current[:cosmo_message]
+    end
+
+    def publish(data, subject, **options)
+      self.class.publish(data, subject:, **options)
+    end
+  end
+end

data/lib/cosmo/utils/hash.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Utils
+    module Hash
+      module_function
+
+      def symbolize_keys!(obj)
+        case obj
+        when ::Hash
+          obj.keys.each do |key|
+            raise ArgumentError, "key cannot be converted to symbol" unless key.respond_to?(:to_sym)
+
+            sym = key.to_sym
+            value = obj.delete(key)
+            obj[sym] = symbolize_keys!(value)
+          end
+          obj
+        when ::Array
+          obj.map! { |v| symbolize_keys!(v) }
+        else
+          obj
+        end
+      end
+
+      # deep dup
+      def dup(hash)
+        Marshal.load(Marshal.dump(hash))
+      end
+
+      # dig keys
+      def keys?(hash, *keys)
+        keys.reduce(hash) do |base, key|
+          return false if !base.is_a?(::Hash) || !base.key?(key)
+
+          base[key]
+        end
+
+        true
+      end
+
+      # deep set
+      def set(hash, *keys, value)
+        last_key = keys.pop
+        target = keys.reduce(hash) do |base, key|
+          base[key] ||= {}
+          base[key]
+        end
+        target[last_key] = value
+      end
+
+      # deep merge
+      def merge(hash1, hash2)
+        return hash1 unless hash2
+
+        hash1.merge(hash2) do |_key, old_val, new_val|
+          if old_val.is_a?(::Hash) && new_val.is_a?(::Hash)
+            merge(old_val, new_val)
+          else
+            new_val
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/utils/json.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Cosmo
+  module Utils
+    module Json
+      module_function
+
+      def parse(value, default: nil, symbolize_names: true, **options)
+        JSON.parse(value, options.merge(symbolize_names:))
+      rescue TypeError, JSON::ParserError
+        default
+      end
+
+      def dump(value, default: nil)
+        ::JSON.generate(value)
+      rescue TypeError, JSON::NestingError
+        default
+      end
+    end
+  end
+end

data/lib/cosmo/utils/signal.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Utils
+    class Signal
+      def self.trap(...)
+        new(...)
+      end
+
+      def initialize(*signals)
+        @queue = Queue.new
+        signals.each { |s| ::Signal.trap(s) { push(s) } }
+      end
+
+      def wait
+        @queue.pop # Wait indefinitely for a signal
+      end
+
+      def push(signal)
+        @queue.push(signal)
+      end
+    end
+  end
+end

data/lib/cosmo/utils/stopwatch.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Utils
+    class Stopwatch
+      def initialize
+        reset
+      end
+
+      # @return [Float] A number of elapsed milliseconds
+      def elapsed_millis
+        (clock_time - @started_at).round(2)
+      end
+
+      # @return [Float] A number of elapsed seconds
+      def elapsed_seconds
+        (elapsed_millis / 1_000).round(2)
+      end
+
+      def reset
+        @started_at = clock_time
+      end
+
+      private
+
+      # @return [Float]
+      def clock_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
+      end
+    end
+  end
+end

data/lib/cosmo/utils/string.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Cosmo
+  module Utils
+    module String
+      module_function
+
+      def underscore(value)
+        value
+          .to_s
+          .gsub("::", "-")
+          .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+          .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+          .downcase
+      end
+
+      def safe_constantize(value)
+        Object.const_get(value)
+      rescue NameError
+        # nop
+      end
+    end
+  end
+end

data/lib/cosmo/utils/thread_pool.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Cosmo
+  module Utils
+    # A thread pool that reuses a fixed number of threads operating off a fixed size queue.
+    # At any point, at most `num_threads` will be active processing tasks. When all threads are busy new
+    # tasks posted to the thread pool are blocked until a thread becomes available.
+    # Should a thread crash for any reason the thread will immediately be removed
+    # from the pool and replaced.
+    class ThreadPool
+      extend Forwardable
+
+      delegate %i[shutdown wait_for_termination] => :@pool
+
+      def initialize(concurrency)
+        @mutex = Thread::Mutex.new
+        @available = concurrency
+        @cond = ConditionVariable.new
+        @pool = Concurrent::FixedThreadPool.new(concurrency)
+      end
+
+      def post
+        @mutex.synchronize do
+          @cond.wait(@mutex) while @available <= 0
+          @available -= 1
+        end
+
+        @pool.post do
+          yield
+        ensure
+          @mutex.synchronize do
+            @available += 1
+            @cond.signal
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cosmo/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Cosmo
+  VERSION = "0.1.0"
+end