artery 1.1.0

data/lib/artery/routing.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Artery
+  module Routing
+    class URI
+      attr_accessor :service, :model, :action, :plural
+
+      def initialize(arg)
+        case arg
+        when URI
+          @service = arg.service
+          @model   = arg.model
+          @action  = arg.action
+          @plural  = arg.plural
+        when String
+          @service, model, @action = arg.split('.').map(&:to_sym)
+          @model = model.to_s.singularize.to_sym
+          @plural = (@model != model)
+        when Hash
+          @service = arg[:service] || Artery.service_name
+          @model   = arg[:model].try(:to_sym)
+          @action  = arg[:action].try(:to_sym)
+          @plural  = arg[:plural]
+        else
+          raise ArgumentError, 'Unknown argument format'
+        end
+        raise(ArgumentError, 'service and model must be provided') if @service.blank? || @model.blank?
+      end
+
+      def to_route
+        [@service, route_model, @action].join('.')
+      end
+      alias to_s to_route
+
+      def plural?
+        @plural
+      end
+
+      def route_model
+        (plural? ? model.to_s.pluralize : model)
+      end
+
+      # Make them identical for Hash if route is identical
+      def ==(other)
+        to_s == other.to_s
+      end
+
+      def eql?(other)
+        self == other
+      end
+
+      def hash
+        to_s.hash
+      end
+    end
+
+    module_function
+
+    def uri(arg)
+      URI.new(arg)
+    end
+  end
+end

data/lib/artery/subscription/incoming_message.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Artery
+  class Subscription
+    class IncomingMessage
+      attr_accessor :data, :reply, :from, :from_uri, :subscription, :options
+
+      def initialize(subscription, data, reply, from, **options)
+        @subscription = subscription
+        @data         = data
+        @attributes   = data[:attributes]
+        @reply        = reply
+        @from         = from
+        @from_uri     = Routing.uri(@from)
+        @options      = options
+      end
+
+      def action
+        from_uri.action
+      end
+
+      def index
+        data[:_index].to_i
+      end
+
+      def previous_index
+        data[:_previous_index].to_i
+      end
+
+      def has_index?
+        index.positive?
+      end
+
+      def from_updates?
+        options[:from_updates]
+      end
+
+      def update_by_us?
+        data[:updated_by_service].to_s == Artery.service_name.to_s
+      end
+
+      def enrich_data # rubocop:disable Metrics/AbcSize
+        # NO enrich needed as we already have message with attributes
+        if @attributes
+          yield @attributes
+          return
+        end
+
+        get_uri = Routing.uri service: from_uri.service,
+                              model: from_uri.model,
+                              plural: true,
+                              action: :get
+        get_data = {
+          uuid: data[:uuid],
+          representation: subscription.representation_name
+        }
+
+        Artery.request get_uri.to_route, get_data do |on|
+          on.success do |attributes|
+            yield attributes
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            error = Error.new("Error in subscription handler: #{e.inspect}",
+                              original_exception: e,
+                              subscription: {
+                                subscriber: subscription.subscriber.to_s,
+                                data: data.to_json,
+                                route: from
+                              },
+                              request: { data: get_data.to_json, route: get_uri.to_route }, response: attributes.to_json)
+            Artery.handle_error error
+          end
+
+          on.error do |e|
+            if e.message == 'not_found'
+              yield(:not_found)
+            else
+              error = Error.new("Failed to get #{get_uri.model} from #{get_uri.service} with uuid='#{data[:uuid]}': #{e.message}",
+                                e.artery_context.merge(subscription: {
+                                                         subscriber: subscription.subscriber.to_s,
+                                                         data: data.to_json,
+                                                         route: from_uri.to_route
+                                                       }))
+              Artery.handle_error error
+            end
+          end
+        end
+      end
+
+      def inspect
+        "<#{from}> <#{reply}> #{data}".inspect
+      end
+    end
+  end
+end

data/lib/artery/subscription/synchronization.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module Artery
+  class Subscription
+    module Synchronization
+      extend ActiveSupport::Concern
+
+      ALIVE_EDGE = 2.minutes
+      HEARTBEAT_INTERVAL = 30.seconds
+
+      def synchronize?
+        options[:synchronize]
+      end
+
+      def synchronize_updates?
+        options[:synchronize_updates]
+      end
+
+      def synchronization_scope
+        options[:synchronize].is_a?(Hash) ? options[:synchronize][:scope] : nil
+      end
+
+      def synchronization_per_page
+        options[:synchronize].is_a?(Hash) ? options[:synchronize][:per_page] : nil
+      end
+
+      def synchronize_updates_scope
+        (options[:synchronize_updates].is_a?(Hash) && options[:synchronize_updates][:scope]) || synchronization_scope
+      end
+
+      def synchronize_updates_per_page
+        (options[:synchronize_updates].is_a?(Hash) && options[:synchronize_updates][:per_page]) || synchronization_per_page
+      end
+
+      def synchronize_updates_autoenrich?
+        options[:synchronize_updates].is_a?(Hash) ? options[:synchronize_updates][:autoenrich] : false
+      end
+
+      def synchronization_in_progress?
+        info.synchronization_in_progress? && synchronization_alive?
+      end
+
+      def synchronization_alive?
+        info.synchronization_heartbeat.blank? || (Time.zone.now - info.synchronization_heartbeat) < ALIVE_EDGE
+      end
+
+      def synchronization_in_progress!(val = true)
+        if val
+          Artery.synchronizing_subscriptions << self
+          run_synchronization_heartbeat
+
+          info.update! synchronization_in_progress: true, synchronization_heartbeat: Time.zone.now
+        else
+          Artery.synchronizing_subscriptions.delete self
+          stop_synchronization_heartbeat
+
+          info.update! synchronization_in_progress: false, synchronization_heartbeat: nil
+        end
+      end
+
+      def synchronization_transaction(&blk)
+        return unless blk
+        return info.synchronization_transaction(&blk) if info.respond_to?(:synchronization_transaction)
+
+        blk.call
+      end
+
+      def synchronization_page_update!(page)
+        info.update! synchronization_page: page
+      end
+
+      def synchronize!
+        return if uri.service == Artery.service_name || synchronization_in_progress?
+
+        if !new? && synchronize_updates?
+          receive_updates
+        elsif new? && synchronize?
+          receive_all
+        end
+      end
+
+      def receive_all
+        synchronization_in_progress! unless synchronization_in_progress?
+
+        while receive_all_once == :continue; end
+      end
+
+      def receive_updates
+        synchronization_in_progress!
+
+        while receive_updates_once == :continue; end
+      end
+
+      private
+
+      def run_synchronization_heartbeat
+        return if @heartbeat_thread
+
+        @heartbeat_thread = Thread.new do
+          while @heartbeat_thread
+            sleep HEARTBEAT_INTERVAL
+
+            info.update! synchronization_heartbeat: Time.zone.now
+          end
+        end
+      end
+
+      def stop_synchronization_heartbeat
+        @heartbeat_thread&.exit
+      end
+
+      def receive_all_once # rubocop:disable Metrics/AbcSize
+        should_continue = false
+        all_uri = Routing.uri(service: uri.service, model: uri.model, plural: true, action: :get_all)
+
+        page = info.synchronization_page ? info.synchronization_page + 1 : 0 if synchronization_per_page
+
+        objects = nil
+
+        all_data = {
+          representation: representation_name,
+          scope: synchronization_scope,
+          page: page,
+          per_page: synchronization_per_page
+        }
+
+        Artery.request all_uri.to_route, all_data do |on|
+          on.success do |data|
+            Artery.logger.debug "HEY-HEY, ALL OBJECTS: <#{all_uri.to_route}> #{[data].inspect}"
+
+            objects = data[:objects].map(&:with_indifferent_access)
+
+            synchronization_transaction { handler.call(:synchronization, objects, page) }
+
+            if synchronization_per_page && objects.count.positive?
+              synchronization_page_update!(page)
+              Artery.logger.debug "PAGE #{page} RECEIVED, WILL CONTINUE..."
+              should_continue = true
+            else
+              synchronization_page_update!(nil) if synchronization_per_page
+              synchronization_in_progress!(false)
+              update_info_by_message!(IncomingMessage.new(self, data, nil, all_uri.to_route))
+            end
+          rescue Exception => e
+            synchronization_in_progress!(false)
+            Artery.handle_error Error.new("Error in all objects request handling: #{e.inspect}",
+                                          original_exception: e,
+                                          request: {
+                                            route: all_uri.to_route,
+                                            data: all_data.to_json
+                                          },
+                                          response: data.to_json)
+          end
+
+          on.error do |e|
+            synchronization_in_progress!(false)
+            error = Error.new "Failed to get all objects #{uri.model} from #{uri.service} with scope='#{synchronization_scope}': " \
+                              "#{e.message}", **e.artery_context
+            Artery.handle_error error
+          end
+        end
+        :continue if should_continue
+      end
+
+      def receive_updates_once # rubocop:disable Metrics/AbcSize
+        should_continue = false
+        updates_uri = Routing.uri(service: uri.service, model: uri.model, plural: true, action: :get_updates)
+        updates_data = {
+          after_index: latest_message_index
+        }
+
+        # Configurable autoenrich updates
+        if synchronize_updates_autoenrich?
+          # we must setup per_page as data is autoenriched and can be big
+          updates_data.merge! representation: representation_name,
+                              scope: synchronize_updates_scope,
+                              per_page: synchronize_updates_per_page
+        end
+
+        Artery.request updates_uri.to_route, updates_data do |on|
+          on.success do |data|
+            Artery.logger.debug "HEY-HEY, LAST_UPDATES: <#{updates_uri.to_route}> #{[data].inspect}"
+
+            updates = data[:updates].map(&:with_indifferent_access)
+            synchronization_transaction do
+              updates.sort_by { |u| u[:_index] }.each do |update|
+                from = Routing.uri(service: uri.service, model: uri.model, action: update.delete(:action)).to_route
+                handle(IncomingMessage.new(self, update, nil, from, from_updates: true))
+              end
+
+              update_info_by_message! IncomingMessage.new(self, data, nil, updates_uri.to_route)
+            end
+
+            if data[:_continue]
+              Artery.logger.debug 'NOT ALL UPDATES RECEIVED, WILL CONTINUE...'
+              should_continue = true
+            else
+              synchronization_in_progress!(false)
+            end
+          rescue Exception => e
+            synchronization_in_progress!(false)
+            Artery.handle_error Error.new("Error in updates request handling: #{e.inspect}",
+                                          original_exception: e,
+                                          request: {
+                                            route: updates_uri.to_route,
+                                            data: updates_data.to_json
+                                          },
+                                          response: data.to_json)
+          end
+
+          on.error do |e|
+            synchronization_in_progress!(false)
+            Artery.handle_error Error.new("Failed to get updates for #{uri.model} from #{uri.service}: #{e.message}",
+                                          **e.artery_context)
+          end
+        end
+        :continue if should_continue
+      end
+    end
+  end
+end

data/lib/artery/subscription.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Artery
+  class Subscription
+    autoload :Synchronization, 'artery/subscription/synchronization'
+    autoload :IncomingMessage, 'artery/subscription/incoming_message'
+
+    include Synchronization
+
+    attr_accessor :uri, :subscriber, :handler, :options
+
+    DEFAULTS = {
+      synchronize: false,
+      synchronize_updates: true,
+      representation: Artery.service_name
+    }.freeze
+
+    def initialize(model, uri, handler:, **options)
+      @uri        = uri
+      @subscriber = model
+      @handler    = handler
+      @options    = DEFAULTS.merge(options)
+
+      Artery.add_subscription self
+    end
+
+    def info
+      @info ||= Artery.subscription_info_class.find_for_subscription(self)
+    end
+
+    def representation_name
+      options[:representation]
+    end
+
+    def latest_message_index
+      info.latest_index.to_i
+    end
+
+    def source?
+      @subscriber.artery[:source]
+    end
+
+    def latest_outgoing_message_index
+      return unless source?
+
+      Artery.message_class.latest_index(@subscriber.artery_model_name)
+    end
+
+    def new?
+      !latest_message_index.positive?
+    end
+
+    def update_info_by_message!(message)
+      return if !message.has_index? || message.from_updates?
+
+      new_data = {}
+      new_data[:latest_index] = message.index if message.index.positive? && (message.index > latest_message_index)
+
+      info.update! new_data
+    end
+
+    def handle(message) # rubocop:disable Metrics/AbcSize
+      Artery.logger.push_tags message.reply
+      Artery.logger.debug "GOT MESSAGE: #{message.inspect}"
+
+      info.lock_for_message(message) do
+        if !message.from_updates? && synchronization_in_progress?
+          Artery.logger.debug 'SKIPPING MESSAGE RECEIVED WHILE SYNC IN PROGRESS'
+          return
+        end
+        return if !message.from_updates? && !validate_index(message)
+
+        if message.update_by_us?
+          Artery.logger.debug 'SKIPPING UPDATE MADE BY US'
+          update_info_by_message!(message)
+          return
+        end
+
+        unless handler.has_block?(message.action) || handler.has_block?(:_default)
+          Artery.logger.debug 'SKIPPING MESSAGE WE ARE NOT LISTENING TO'
+          update_info_by_message!(message)
+          return
+        end
+
+        case message.action
+        when :create, :update
+          message.enrich_data do |attributes|
+            handle_data(message, attributes)
+          end
+        else
+          handle_data(message)
+        end
+      end
+    ensure
+      Artery.logger.pop_tags
+    end
+
+    protected
+
+    def validate_index(message)
+      return true unless message.previous_index.positive? && latest_message_index.positive?
+
+      if message.previous_index > latest_message_index
+        Artery.logger.debug 'WE\'VE GOT FUTURE MESSAGE, REQUESTING ALL MISSED'
+
+        synchronize! # this will include current message
+        false
+      elsif message.previous_index < latest_message_index
+        Artery.logger.debug 'WE\'VE GOT PREVIOUS MESSAGE AND ALREADY HANDLED IT, SKIPPING'
+
+        false
+      else
+        true
+      end
+    end
+
+    def handle_data(message, data = nil)
+      data ||= message.data
+
+      info.lock_for_message(message) do
+        if data == :not_found # special data when enrich failed with not_found
+          Artery.logger.debug 'SKIP HANDLING MESSAGE BECAUSE ENRICH DATA IS BLANK'
+        else
+          handler.call(:_before_action, message.action, data, message.reply, message.from)
+
+          handler.call(message.action,  data, message.reply, message.from) ||
+            handler.call(:_default, data, message.reply, message.from)
+
+          handler.call(:_after_action, message.action, data, message.reply, message.from)
+        end
+
+        update_info_by_message!(message)
+      end
+    end
+  end
+end

data/lib/artery/subscriptions.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Artery
+  autoload :Subscription,        'artery/subscription'
+  autoload :HealthzSubscription, 'artery/healthz_subscription'
+  autoload :WorkerHealthzSubscription, 'artery/worker_healthz_subscription'
+
+  module Subscriptions
+    extend ActiveSupport::Concern
+    included do
+      class << self
+        attr_accessor :subscriptions
+      end
+    end
+
+    module ClassMethods
+      def subscriptions_on(*services)
+        services = services.flatten.map(&:to_sym)
+
+        subscriptions.slice(*subscriptions.keys.select { |uri| services.include?(uri.service) })
+      end
+
+      def add_subscription(subscription)
+        @subscriptions ||= {}
+        @subscriptions[subscription.uri] ||= []
+        @subscriptions[subscription.uri] << subscription
+      end
+
+      def synchronizing_subscriptions
+        @synchronizing_subscriptions ||= []
+      end
+
+      def clear_synchronizing_subscriptions!
+        Artery.synchronizing_subscriptions.dup.each do |s|
+          Artery.logger.warn "<#{s.subscriber}> [#{s.uri}] is still synchronizing, clearing.."
+
+          s.synchronization_in_progress!(false)
+        end
+      end
+    end
+  end
+end

data/lib/artery/sync.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Artery
+  class Sync
+    attr_accessor :sync_id
+
+    def initialize(sync_id)
+      @sync_id = sync_id
+    end
+
+    def execute(services = nil)
+      services = Array.wrap(services).map(&:to_sym)
+      subscriptions_on_services = services.blank? ? Artery.subscriptions : Artery.subscriptions_on(services)
+
+      if subscriptions_on_services.blank?
+        Artery.logger.warn 'No suitable subscriptions defined, exiting...'
+        return
+      end
+
+      @sync_fiber = Fiber.new do # all synchronization inside must be synchronous
+        subscriptions_on_services.values.flatten.uniq.each(&:synchronize!)
+      end
+      @sync_fiber.resume
+    end
+
+    def self.run(subscriptions)
+      sync_id = SecureRandom.hex
+      Artery.logger.push_tags('Sync', sync_id)
+      Artery::Sync.new(sync_id).execute subscriptions
+    ensure
+      Artery.clear_synchronizing_subscriptions!
+      Artery.logger.pop_tags
+    end
+  end
+end

data/lib/artery/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Artery
+  VERSION = '1.1.0'
+end

data/lib/artery/worker.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Artery
+  class Worker
+    class Error < Artery::Error; end
+
+    attr_reader :worker_id
+
+    def initialize
+      @worker_id = SecureRandom.hex
+    end
+
+    def subscribe_healthz
+      HealthzSubscription.new.subscribe
+      WorkerHealthzSubscription.new(worker_id, 'worker').subscribe
+    end
+
+    def run(services = nil)
+      services = Array.wrap(services).map(&:to_sym)
+      subscriptions_on_services = services.blank? ? Artery.subscriptions : Artery.subscriptions_on(services)
+
+      if subscriptions_on_services.blank?
+        Artery.logger.warn 'No suitable subscriptions defined, exiting...'
+        return
+      end
+
+      Artery.handle_signals
+
+      @sync = Artery::Sync.new worker_id
+
+      Artery.worker = self
+      Artery.start { worker_cycle(services, subscriptions_on_services) }
+    ensure
+      Artery.clear_synchronizing_subscriptions!
+    end
+
+    private
+
+    def worker_cycle(services, subscriptions_on_services)
+      Artery.logger.push_tags 'Worker', worker_id
+      tries = 0
+      begin
+        subscribe_healthz
+
+        @sync.execute services
+
+        subscriptions_on_services.each do |uri, subscriptions|
+          Artery.logger.debug "Subscribing on '#{uri}'"
+          Artery.subscribe uri.to_route, queue: "#{Artery.service_name}.worker" do |data, reply, from|
+            subscriptions.each do |subscription|
+              message = Subscription::IncomingMessage.new subscription, data, reply, from
+
+              subscription.handle(message)
+            rescue StandardError => e
+              Artery.handle_error Error.new("Error in subscription handling: #{e.inspect}",
+                                            original_exception: e,
+                                            subscription: {
+                                              subscriber: subscription.subscriber.to_s,
+                                              route: from,
+                                              data: data.to_json
+                                            })
+            end
+          end
+        end
+      rescue StandardError => e
+        tries += 1
+
+        Artery.handle_error Error.new("WORKER ERROR: #{e.inspect}", original_exception: e)
+        retry if tries <= 5
+
+        Artery.handle_error Error.new('Worker failed 5 times and exited.')
+      end
+    end
+  end
+end

data/lib/artery/worker_healthz_subscription.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Artery
+  class WorkerHealthzSubscription
+    attr_reader :id, :name
+
+    def initialize(id, name)
+      @id = id
+      @name = name
+    end
+
+    def subscribe
+      healthz_route = "#{Artery.service_name}.healthz.#{name}"
+      Artery.logger.debug "Subscribing on '#{healthz_route}' for #{name} #{id}"
+
+      Artery.subscribe healthz_route do |data, reply, _from|
+        next unless data['id'] == id
+
+        Artery.publish reply, status: :ok
+      end
+    end
+  end
+end