yes-core 1.0.0

data/lib/yes/core/read_model/builder.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+require 'i18n'
+
+begin
+  require 'trx_ext'
+rescue LoadError
+end
+
+module Yes
+  module Core
+    module ReadModel
+      # Base class for read model builders that distribute events to proper event handlers
+      # and manage read model lifecycle including rebuilding.
+      class Builder
+        include OpenTelemetry::Trackable
+
+        EventValidationErrorsPresent = Class.new(Error)
+        InvalidReadModelBuilderClass = Class.new(Error)
+        MissingReadModelId = Class.new(Error)
+        RebuildError = Class.new(Error)
+
+        READ_MODEL_CLASS_REGEXP =
+          /(?<context>\w+)::ReadModels::(?<version>V\d+)?(::)?(?<aggregate>\w+)/
+
+        # Based on event type class it distributes event to a proper event handler
+        # @param event [Yes::Core::Event]
+        # @param read_model [ActiveRecord::Base] AR object
+        # @return [void]
+        def call(event, read_model: nil)
+          read_model ||= read_model(event)
+
+          otl_record_data(read_model, event)
+
+          handler = handler_class(event)&.new(read_model)
+          return unless handler
+
+          locale = event.data['locale']&.to_sym || I18n.default_locale
+          return unless correct_locale?(locale)
+
+          I18n.with_locale(locale) do
+            next handler.call(event) unless self.class.otl_tracer
+
+            OpenTelemetry::OtlSpan.new(otl_data: otl_data(handler), otl_tracer: self.class.otl_tracer).otl_span do
+              handler.call(event)
+            end
+          ensure
+            otl_record_processed(read_model, event, handler)
+          end
+        end
+        otl_trackable :call, OpenTelemetry::OtlSpan::OtlData.new(
+          span_kind: :consumer,
+          links_extractor: ->(event, **) { event.metadata['otl_contexts'] },
+          track_sql: true
+        )
+
+        # @param eventstore [PgEventstore::Client] eventstore client
+        # @param ids [Array<String>] ids to rebuild
+        def rebuild_many(
+          eventstore: PgEventstore.client,
+          ids: read_model_class.pluck(:id)
+        )
+          ids.each { |id| rebuild(eventstore:, id:) }
+        end
+
+        # @param id [String] id of the read model to rebuild
+        # @param eventstore [PgEventstore::Client] eventstore client
+        # @return [ActiveRecord::Base, nil]
+        # @raise [Yes::Core::ReadModel::Builder::RebuildError]
+        def rebuild(id:, eventstore: PgEventstore.client)
+          # delete is intentional to avoid any callbacks
+          read_model_class.delete(id) if read_model_class.exists?(id)
+          context = read_model_class_name_match[:context]
+          stream_name = read_model_class_name_match[:aggregate]
+          enum = eventstore.read_paginated(
+            PgEventstore::Stream.new(context:, stream_name:, stream_id: id), options: { resolve_link_tos: true }
+          )
+          read_model = read_model_class.find_or_create_by(id:)
+
+          enum.each do |events|
+            events.each do |event|
+              # pass in read model for efficiency
+              call(event, read_model:)
+            end
+          end
+          read_model
+        end
+
+        # @return [String]
+        def aggregate_id_key
+          "#{underscore(read_model_class_name_match[:aggregate].to_s)}_id"
+        end
+
+        def initialize
+          self.read_model_class = default_read_model_class
+        end
+
+        protected
+
+        attr_accessor :read_model_class
+
+        # @param event [Yes::Core::Event] event to find handler for
+        # @return [Class, nil] handler class for the event
+        def handler_class(event)
+          handler_classes = possible_handler_class_names(event)
+          handler_instance = handler_classes.lazy.filter_map do |class_name|
+            Kernel.const_get(class_name)
+          rescue NameError
+            next
+          end.first
+
+          notify_missing_event_handler(event, handler_classes) unless handler_instance
+
+          handler_instance
+        end
+
+        # @param event [Yes::Core::Event] event to find handler names for
+        # @return [Array<String>] possible handler class names
+        def possible_handler_class_names(event)
+          parent_modules = parent_modules_string
+          event_type = event_type_name(event)
+          event_context = event_context_name(event)
+          # in case event has no context
+          return ["#{parent_modules}::On#{event_type}"] if event_type == event_context
+
+          [
+            "#{parent_modules}::#{event_context}::On#{event_type}",
+            "#{parent_modules}::On#{event_type}"
+          ]
+        end
+
+        # @param event [Yes::Core::Event] event that has no handler
+        # @param possible_handler_classes_name [Array<String>] handler class names that were tried
+        def notify_missing_event_handler(event, possible_handler_classes_name)
+          msg = "The event handler #{possible_handler_classes_name} is not defined."
+          Utils::ErrorNotifier.new.event_handler_not_defined(msg, event)
+        end
+
+        # @param event [Yes::Core::Event] event to extract type name from
+        # @return [String]
+        def event_type_name(event)
+          event.type.split('::').last
+        end
+
+        # @param event [Yes::Core::Event] event to extract context name from
+        # @return [String]
+        def event_context_name(event)
+          event.type.split('::').first
+        end
+
+        # @return [String]
+        def parent_modules_string
+          self.class.to_s.split('::')[0..-2].join('::')
+        end
+
+        # @return [Class] default read model class inferred from builder class name
+        # @raise [InvalidReadModelBuilderClass] if class name doesn't match expected pattern
+        def default_read_model_class
+          match = read_model_class_name_match
+          klass = [match[:version], match[:aggregate]].compact.join('::')
+          Kernel.const_get klass
+        rescue NameError, TypeError
+          msg = "The Read Model Builder Class #{self.class} does not match modules structure"
+          raise InvalidReadModelBuilderClass, msg
+        end
+
+        private
+
+        # @return [MatchData] match data from class name regex
+        def read_model_class_name_match
+          @read_model_class_name_match ||=
+            READ_MODEL_CLASS_REGEXP.match(self.class.to_s)
+        end
+
+        # @param locale [Symbol] locale to check
+        # @return [Boolean] true if locale is available
+        def correct_locale?(locale)
+          return true unless locale
+
+          I18n.available_locales.map(&:to_sym).include?(locale.to_sym)
+        end
+
+        # @param event [Yes::Core::Event] event to find read model for
+        # @return [ActiveRecord::Base] the read model instance
+        # @raise [MissingReadModelId] if event data doesn't contain aggregate id
+        def read_model(event)
+          read_model_id = event.data[aggregate_id_key]
+          raise MissingReadModelId.new(extra: event) unless read_model_id
+
+          find_or_create_read_model(read_model_id)
+        end
+
+        if defined?(TrxExt)
+          def find_or_create_read_model(read_model_id)
+            read_model_class.trx do
+              read_model_class.find_by(id: read_model_id) || read_model_class.create(id: read_model_id)
+            end
+          end
+        else
+          def find_or_create_read_model(read_model_id)
+            read_model_class.create_or_find_by!(id: read_model_id)
+          end
+        end
+
+        # derived from https://api.rubyonrails.org/classes/ActiveSupport/Inflector.html#method-i-underscore
+        # @param word [String] word to underscore
+        # @return [String] underscored word
+        def underscore(word)
+          w = word.dup
+          w.gsub!(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2')
+          w.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
+          w.tr!('-', '_')
+          w.downcase
+        end
+
+        # @param read_model [ActiveRecord::Base] the read model
+        # @param event [Yes::Core::Event] the event
+        # @param handler [Yes::Core::ReadModel::EventHandler] the handler that processed the event
+        def otl_record_processed(_read_model, event, handler)
+          return if ENV['STATSD_ADDR'].blank?
+
+          StatsD.increment(
+            'events_processing_total',
+            tags: {
+              type: 'consumer',
+              service: Rails.application.class.module_parent.name,
+              source: "#{event.metadata.dig('otl_contexts', 'root', 'service') || 'unknown'}-#{event.type}",
+              target: "#{Rails.application.class.module_parent.name}-#{self.class.name}",
+              read_model_builder_name: self.class.name,
+              event_handler_name: handler.class.name,
+              event: event.type
+            }
+          )
+        end
+
+        # @param read_model [ActiveRecord::Base] the read model
+        # @param event [Yes::Core::Event] the event
+        def otl_record_data(read_model, event)
+          return unless self.class.otl_tracer
+          return if event.created_at.blank?
+
+          span_time = (Time.at(0, self.class.current_span.start_timestamp, :nanosecond).to_f * 1000).to_i
+          event_publish_delay_ms = span_time - (event.created_at.utc.to_f * 1000).to_i
+
+          self.class.current_span&.add_attributes(
+            {
+              'event_published_delay_ms' => event_publish_delay_ms,
+              'event_published_at_ms' => (event.created_at.to_f * 1000).to_i,
+              'command_request_started_at_ms' => event.metadata.dig('otl_contexts', 'timestamps',
+                                                                    'command_request_started_at_ms'),
+              'command_handling_started_at_ms' => event.metadata.dig('otl_contexts', 'timestamps',
+                                                                     'command_handling_started_at_ms'),
+              'read_model_builder.name' => self.class.name,
+              'read_model_builder.read_model.class' => read_model.class.name
+            }.compact_blank
+          )
+        end
+
+        # @param handler [Yes::Core::ReadModel::EventHandler] handler to build OTL data for
+        # @return [Yes::Core::OpenTelemetry::OtlSpan::OtlData]
+        def otl_data(handler)
+          OpenTelemetry::OtlSpan::OtlData.new(span_name: handler.class.name, span_kind: :consumer, track_sql: true)
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/read_model/event_handler.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module ReadModel
+      # Base class for event handlers that process events and update read models.
+      class EventHandler
+        include OpenTelemetry::Trackable
+
+        attr_accessor :read_model, :payload_store_lookup
+        private :read_model, :payload_store_lookup, :read_model=, :payload_store_lookup=
+
+        # @param read_model [ActiveRecord::Base] AR object
+        # @param payload_store_lookup [#call] payload store lookup instance
+        def initialize(
+          read_model,
+          payload_store_lookup: Yes::Core::PayloadStore::Lookup.new
+        )
+          raise ArgumentError unless read_model
+
+          self.read_model = read_model
+          self.payload_store_lookup = payload_store_lookup
+        end
+
+        # @param event [Yes::Core::Event] event to handle
+        # @return [Yes::Core::Event] the processed event
+        def call(event)
+          otl_record_event_data(event) if self.class.otl_tracer
+
+          payload_store_lookup.call(event).each do |key, value|
+            event.data[key.to_s] = value
+          end
+
+          event
+        end
+        otl_trackable :call, OpenTelemetry::OtlSpan::OtlData.new(span_kind: :consumer, track_sql: true)
+
+        private
+
+        # @param event [Yes::Core::Event] event to record telemetry data for
+        def otl_record_event_data(event)
+          return if event.created_at.blank?
+
+          span_time = (Time.at(0, self.class.current_span.start_timestamp, :nanosecond).to_f * 1000).to_i
+          event_publish_delay_ms = span_time - (event.created_at.utc.to_f * 1000).to_i
+
+          self.class.current_span&.add_attributes(
+            {
+              'event_published_delay_ms' => event_publish_delay_ms,
+              'event_published_at_ms' => (event.created_at.utc.to_f * 1000).to_i,
+              'command_request_started_at_ms' => event.metadata.dig('otl_contexts', 'timestamps',
+                                                                    'command_request_started_at_ms'),
+              'command_handling_started_at_ms' => event.metadata.dig('otl_contexts', 'timestamps',
+                                                                     'command_handling_started_at_ms'),
+              'event.type' => event.type,
+              'event.data' => event.data.to_json,
+              'event.metadata' => event.metadata.to_json
+            }.compact_blank
+          )
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/read_model/filter.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'has_scope'
+
+module Yes
+  module Core
+    module ReadModel
+      # Inherit from this class to define your implementation. Your class should implement #read_model_class instance
+      # method should determine what ActiveRecord model class to use.
+      # Example usage:
+      #   ```ruby
+      #   class JobAppFilter < Yes::Core::ReadModel::Filter
+      #     has_scope :ids do |_controller, scope, value|
+      #       scope.where(id: value.split(','))
+      #     end
+      #     has_scope :order_by_name do |_controller, scope, value|
+      #       scope.order(name: value)
+      #     end
+      #     has_scope :order_by_id do |_controller, scope, value|
+      #       scope.order(id: value)
+      #     end
+      #
+      #     private
+      #
+      #     def read_model_class
+      #       ::JobApp
+      #     end
+      #   end
+      #   ```
+      #
+      # Now you can use it as follows:
+      #   ```ruby
+      #   JobAppFilter.new(order: { name: 'asc', id: 'desc' }, filters: { ids: '1,2,3' }).call
+      #   ```
+      class Filter
+        include HasScope
+        include FilterQueryBuilder
+
+        ORDER_DIRECTIONS = {
+          'asc' => 'asc', 'desc' => 'desc', '0' => 'asc', '1' => 'desc'
+        }.tap { |h| h.default = 'asc' }.freeze
+
+        attr_accessor :options, :type
+        private :options, :type
+
+        # Returns scope of persisted filters if given filter supports saving.
+        # e. g. `AdvancedFilter.where(read_model: 'apprenticeships')`
+        # @return [ActiveRecord::Relation]
+        def self.persisted_filter_scope
+          raise NotImplementedError
+        end
+
+        # @param options [Hash]
+        # @option [Hash] :order defines an order of records of your query.
+        #   Example: { order: { name: 'asc', id: 'desc' } }.
+        #   Note: you should have `:order_by_name` and `:order_by_id` scopes defined using `#has_scope` class method
+        # @option [Hash] :filters defines a filtering of records of your query.
+        #   Example: { filters: { ids: '1,2,3', name: 'Awasome-4000' } }
+        #   Note: you should have `:ids` scope defined using `#has_scope` class method
+        # @option [Hash] :filter_definition defines an advanced filter supporting linking multiple filters with AND/OR
+        #   logic.
+        #   Example:
+        # {
+        #   filter_definition: {
+        #     type: 'filter_set',
+        #     logical_operator: 'or',
+        #     filters: [
+        #       { type: 'filter', attribute: 'first_name', operator: 'is', value: 'Foo' },
+        #       { type: 'filter', attribute: 'last_name', operator: 'is_not', value: 'Bar' }
+        #     ]
+        #   }
+        # }
+        # @param [Symbol] :type defines the type of filter to use, default is :basic. Passing :advanced is required to
+        #   use the :filter_definition option.
+        def initialize(options, type: :basic)
+          @options = options
+          @type = type
+        end
+
+        # @return [ActiveRecord::Relation]
+        def call
+          scope = read_model_class.order(id: :asc)
+          if type == :basic
+            scope = apply_scopes(scope, options[:filters]) if options[:filters].is_a?(Hash)
+          else
+            scope = process_advanced_filter(scope, options[:filter_definition])
+          end
+          scope = apply_scopes(scope.reorder(nil), ordering_options(options[:order])) if options[:order].is_a?(Hash)
+          scope
+        end
+
+        private
+
+        # Transforms a hash into a set of scopes names and normalized directions.
+        # Example:
+        #   Given options:
+        #   ```ruby
+        #   { name: '1' }
+        #   ```
+        #   Will result in:
+        #   ```ruby
+        #   { order_by_name: 'desc' }
+        #   ```
+        # @param options [Hash] order options
+        # @return [Hash]
+        def ordering_options(options)
+          options.each_with_object({}) do |(column_name, direction), result|
+            result[:"order_by_#{column_name}"] = ORDER_DIRECTIONS[direction]
+          end
+        end
+
+        def read_model_class
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/read_model/filter_query_builder.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module ReadModel
+      # Provides advanced filter query building capabilities using AND/OR logic.
+      # Include this module in filter classes that need to support advanced filtering.
+      module FilterQueryBuilder
+        AND_LOGICAL_OPERATOR = 'and'
+        OR_LOGICAL_OPERATOR = 'or'
+        IS_OPERATOR = 'is'
+        IS_NOT_OPERATOR = 'is_not'
+
+        FilterSchema = Dry::Schema.Params do
+          required(:type).value(eql?: 'filter')
+          required(:attribute).filled(:string)
+          required(:operator).value(included_in?: [IS_NOT_OPERATOR, IS_OPERATOR])
+          required(:value).value(:any)
+        end
+
+        FilterSetSchema = Dry::Schema.Params do
+          required(:type).value(eql?: 'filter_set')
+          required(:logical_operator).value(included_in?: [AND_LOGICAL_OPERATOR, OR_LOGICAL_OPERATOR])
+          required(:filters).array(FilterSchema)
+          optional(:scope).value(:hash)
+        end
+
+        private
+
+        # Builds an advanced filter query based on the filter definition.
+        #
+        # @param base_advanced_filter_scope [ActiveRecord::Relation] The base scope to apply the filter to.
+        # @param filter_definition [Hash] The filter definition to use.
+        #
+        # @return [ActiveRecord::Relation]
+        def process_advanced_filter(base_advanced_filter_scope, filter_definition)
+          return base_advanced_filter_scope if filter_definition.blank?
+          raise 'Invalid filter definition' unless FilterSetSchema.call(filter_definition).success?
+
+          @base_advanced_filter_scope = base_advanced_filter_scope
+          @filter_definition = filter_definition.deep_symbolize_keys
+          apply_main_scope
+
+          return build_and_query if @filter_definition[:logical_operator] == AND_LOGICAL_OPERATOR
+
+          build_or_query
+        end
+
+        # Chains multiple filters with AND logic.
+        #
+        # @return [ActiveRecord::Relation]
+        def build_and_query
+          scope = @base_advanced_filter_scope
+          @filter_definition[:filters].each do |filter|
+            scope =
+              if filter[:operator] == IS_OPERATOR
+                apply_is_filter(scope, filter)
+              else
+                apply_is_not_filter(scope, filter)
+              end
+          end
+          scope
+        end
+
+        # Chains multiple filters with OR logic.
+        #
+        # @return [ActiveRecord::Relation]
+        def build_or_query
+          scopes = @filter_definition[:filters].map do |filter|
+            if filter[:operator] == IS_OPERATOR
+              apply_is_filter(@base_advanced_filter_scope, filter)
+            else
+              apply_is_not_filter(@base_advanced_filter_scope, filter)
+            end
+          end
+
+          scope = scopes.first
+          scopes[1..].each do |additional_scope|
+            scope = scope.or(additional_scope)
+          end
+
+          scope
+        end
+
+        # Applies given scope to the base advanced filter scope to support authorization.
+        def apply_main_scope
+          return if @filter_definition[:scope].blank?
+
+          @base_advanced_filter_scope = apply_scopes(@base_advanced_filter_scope, @filter_definition[:scope])
+        end
+
+        # Applies a single filter to the scope.
+        def apply_is_filter(scope, filter)
+          apply_scopes(scope, { filter[:attribute].to_sym => filter[:value] })
+        end
+
+        # Applies negation of a single filter to the scope.
+        def apply_is_not_filter(scope, filter)
+          scope.where.not(id: apply_is_filter(@base_advanced_filter_scope, filter))
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/serializer.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'jsonapi/serializer'
+
+module Yes
+  module Core
+    # Base JSON:API serializer for read models.
+    #
+    # All auto-generated and user-defined serializers inherit from this class.
+    # Wraps the jsonapi-serializer gem.
+    #
+    # @example
+    #   class UserSerializer < Yes::Core::Serializer
+    #     set_type 'users'
+    #     attributes :id, :email, :first_name, :last_name
+    #   end
+    class Serializer
+      include JSONAPI::Serializer
+    end
+  end
+end

data/lib/yes/core/subscriptions.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'net/http'
+
+module Yes
+  module Core
+    # Manages PgEventstore subscriptions with optional heartbeat and OpenTelemetry tracing.
+    #
+    # @example
+    #   subscriptions = Yes::Core::Subscriptions.new
+    #   subscriptions.subscribe_to_all(handler, filter_opts)
+    #   subscriptions.start
+    class Subscriptions
+      include Yes::Core::OpenTelemetry::Trackable
+
+      # @return [Integer] Timeout in seconds for subscriptions to start
+      SUBSCRIPTIONS_START_TIMEOUT = 20
+
+      # @return [PgEventstore::SubscriptionsManager] the subscriptions manager
+      attr_reader :subscriptions_manager
+
+      # @return [PgEventstore::Client] the event store client
+      attr_reader :client
+
+      # Initializes subscriptions with the given PgEventstore config.
+      #
+      # @param config_name [Symbol] the PgEventstore configuration name
+      def initialize(config_name: :default)
+        @subscriptions_manager = PgEventstore.subscriptions_manager(
+          config_name,
+          subscription_set: Rails.application.class.name.split('::').first
+        )
+        @client = PgEventstore.client(config_name)
+      end
+
+      # Subscribes a handler to all events matching the given filter.
+      #
+      # @param handler [#call] the event handler
+      # @param filter_opts [Hash] PgEventstore filter options
+      # @param subscription_opts [Hash] additional subscription options
+      # @return [void]
+      def subscribe_to_all(handler, filter_opts, **subscription_opts)
+        subscriptions_manager.subscribe(
+          handler.class.to_s,
+          handler: self.class.otl_tracer ? otl_trackable_handler(handler) : handler,
+          options: { filter: filter_opts, resolve_link_tos: true },
+          **subscription_opts
+        )
+      end
+
+      # Starts the subscriptions manager and optional heartbeat.
+      #
+      # @return [void]
+      def start
+        start_heartbeat if Yes::Core.configuration.subscriptions_heartbeat_url.present?
+        subscriptions_manager.start
+      end
+
+      private
+
+      # Wraps a handler with OpenTelemetry tracing.
+      #
+      # @param handler [#call] the event handler to wrap
+      # @return [Proc] the wrapped handler
+      def otl_trackable_handler(handler)
+        proc do |event|
+          otl_data = Yes::Core::OpenTelemetry::OtlSpan::OtlData.new(
+            span_name: handler.class.name,
+            span_kind: :consumer,
+            links_extractor: ->(ev, **) { ev.metadata['otl_contexts'] },
+            track_sql: true
+          )
+          Yes::Core::OpenTelemetry::OtlSpan.new(otl_data:, otl_tracer: self.class.otl_tracer).
+            otl_span(event) { handler.call(event) }
+        end
+      end
+
+      # Starts a background thread that periodically sends a heartbeat HTTP GET request.
+      #
+      # @return [Thread] the heartbeat thread
+      def start_heartbeat
+        Thread.new do
+          loop do
+            sleep Yes::Core.configuration.subscriptions_heartbeat_interval
+            uri = URI(Yes::Core.configuration.subscriptions_heartbeat_url)
+            Net::HTTP.get(uri)
+          rescue StandardError => e
+            Rails.logger.warn("Subscriptions heartbeat failed: #{e.message}") if defined?(Rails)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/test_support/event_helpers.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module TestSupport
+      # Helpers for working with PgEventstore events in tests.
+      module EventHelpers
+        # @param stream [PgEventstore::Stream]
+        # @param event [Yes::Core::Event]
+        # @return [Yes::Core::Event]
+        def append_and_reload_event(stream, event)
+          PgEventstore.client.append_to_stream(stream, event)
+          PgEventstore.client.read(stream, options: { max_count: 1, direction: :desc }).first
+        end
+
+        # Reads eventstore and returns events from the stream or an empty array if stream does not exist
+        # @param stream [PgEventstore::Stream]
+        # @return [Array<Yes::Core::Event>]
+        def safe_read(stream)
+          PgEventstore.client.read(stream)
+        rescue PgEventstore::StreamNotFoundError
+          []
+        end
+      end
+    end
+  end
+end