yes-core 1.0.0

data/lib/yes/core/jobs/read_model_recovery_job.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module Jobs
+      # Background job that runs periodically to recover stuck read models
+      # This job should be scheduled to run every 30 seconds via cron or similar
+      class ReadModelRecoveryJob < ActiveJob::Base
+        # Circuit breaker configuration
+        MAX_CONSECUTIVE_FAILURES = 5
+        CIRCUIT_BREAKER_TIMEOUT = 5.minutes
+
+        queue_as :critical
+
+        class << self
+          # Track circuit breaker state in memory (or Redis in production)
+          attr_accessor :consecutive_failures, :circuit_opened_at
+        end
+
+        self.consecutive_failures = 0
+        self.circuit_opened_at = nil
+
+        # Performs the recovery of stuck read models
+        # @param stuck_timeout [Integer] Minutes after which a model is considered stuck (default: 1)
+        # @param batch_size [Integer] Number of read models to process at once (default: 100)
+        def perform(stuck_timeout_minutes: 1, batch_size: 100)
+          # Check circuit breaker
+          if circuit_open?
+            Rails.logger.warn('ReadModelRecoveryJob circuit breaker is open, skipping execution')
+            return
+          end
+
+          stuck_timeout = stuck_timeout_minutes.minutes
+
+          Rails.logger.info("Starting read model recovery scan (timeout: #{stuck_timeout_minutes} minutes)")
+
+          results = Yes::Core::CommandHandling::ReadModelRecoveryService.recover_all_stuck_read_models(
+            stuck_timeout:,
+            batch_size:
+          )
+
+          process_results(results)
+
+          # Reset circuit breaker on success
+          self.class.consecutive_failures = 0
+
+          Rails.logger.info("Read model recovery scan completed: #{results.size} models processed")
+        rescue ActiveRecord::ActiveRecordError => e
+          # Database-related errors (connection issues, deadlocks, etc.)
+          Rails.logger.error("Database error during read model recovery: #{e.message}")
+          handle_job_failure(e)
+          raise
+        rescue PgEventstore::Error => e
+          # Event store related errors (revision conflicts, stream errors)
+          Rails.logger.error("Event store error during read model recovery: #{e.message}")
+          handle_job_failure(e)
+          raise
+        rescue NameError, NoMethodError => e
+          # Configuration or class loading errors - these should not trigger circuit breaker
+          Rails.logger.error("Configuration error during read model recovery: #{e.message}")
+          Rails.logger.error('This likely indicates a misconfiguration - please check your read model classes')
+          # Don't increment circuit breaker for configuration errors
+          raise
+        rescue StandardError => e
+          # Unexpected errors - log with full backtrace for debugging
+          Rails.logger.error("Unexpected error during read model recovery: #{e.class.name} - #{e.message}")
+          Rails.logger.error(e.backtrace.join("\n"))
+
+          # Still handle as a failure for circuit breaker
+          handle_job_failure(e)
+
+          # Re-raise to ensure job framework knows it failed
+          raise
+        end
+
+        private
+
+        # Processes the recovery results and tracks metrics
+        # @param results [Array<RecoveryResult>] The recovery results
+        def process_results(results)
+          successful = results.count(&:success)
+          failed = results.count { |r| !r.success }
+
+          if failed.positive?
+            Rails.logger.warn("Read model recovery had #{failed} failures out of #{results.size} attempts")
+
+            # Send alert if too many failures
+            alert_on_high_failure_rate(failed, results.size) if failed > results.size / 2
+          end
+
+          # Track metrics (integrate with your monitoring solution)
+          track_metrics(successful:, failed:)
+
+          # Alert on models stuck for too long
+          check_for_long_stuck_models
+        end
+
+        # Checks if circuit breaker is open
+        # @return [Boolean] True if circuit is open
+        def circuit_open?
+          return false unless self.class.circuit_opened_at
+
+          if Time.current - self.class.circuit_opened_at > CIRCUIT_BREAKER_TIMEOUT
+            Rails.logger.info('ReadModelRecoveryJob circuit breaker timeout expired, resetting')
+            self.class.circuit_opened_at = nil
+            self.class.consecutive_failures = 0
+            false
+          else
+            true
+          end
+        end
+
+        # Handles job failure and manages circuit breaker
+        # @param error [Exception] The error that occurred
+        def handle_job_failure(error)
+          self.class.consecutive_failures += 1
+
+          Rails.logger.error(
+            "ReadModelRecoveryJob failed (attempt #{self.class.consecutive_failures}): #{error.message}"
+          )
+
+          return unless self.class.consecutive_failures >= MAX_CONSECUTIVE_FAILURES
+
+          self.class.circuit_opened_at = Time.current
+          Rails.logger.error(
+            "ReadModelRecoveryJob circuit breaker opened after #{MAX_CONSECUTIVE_FAILURES} consecutive failures"
+          )
+
+          # Send critical alert
+          alert_on_circuit_breaker_open
+        end
+
+        # Checks for models that have been stuck for too long
+        def check_for_long_stuck_models
+          critical_timeout = 5.minutes
+
+          # Find models stuck for more than critical timeout
+          long_stuck_models = find_long_stuck_models(critical_timeout)
+
+          return unless long_stuck_models.any?
+
+          alert_on_long_stuck_models(long_stuck_models)
+        end
+
+        # Finds read models stuck for longer than the specified timeout
+        # @param timeout [ActiveSupport::Duration] The timeout duration
+        # @return [Array<ActiveRecord::Base>] Long stuck models
+        def find_long_stuck_models(timeout)
+          models = []
+
+          read_model_classes.each do |model_class|
+            next unless model_class.column_names.include?('pending_update_since')
+
+            models.concat(
+              model_class.
+                where.not(pending_update_since: nil).
+                where(pending_update_since: ...timeout.ago).
+                to_a
+            )
+          end
+
+          models
+        end
+
+        # Gets all read model classes from configuration
+        # @return [Array<Class>] Read model classes
+        def read_model_classes
+          Yes::Core.configuration.all_read_model_classes
+        end
+
+        # Tracks metrics (implement based on your monitoring solution)
+        # @param successful [Integer] Number of successful recoveries
+        # @param failed [Integer] Number of failed recoveries
+        def track_metrics(successful:, failed:)
+          # Example for StatsD/DataDog
+          # StatsD.gauge('read_model_recovery.successful', successful)
+          # StatsD.gauge('read_model_recovery.failed', failed)
+
+          # Example for Prometheus
+          # Prometheus.gauge(:read_model_recovery_successful, successful)
+          # Prometheus.gauge(:read_model_recovery_failed, failed)
+
+          Rails.logger.info("Recovery metrics - Successful: #{successful}, Failed: #{failed}")
+        end
+
+        # Sends alert on high failure rate
+        # @param failed [Integer] Number of failures
+        # @param total [Integer] Total attempts
+        def alert_on_high_failure_rate(failed, total)
+          message = "High read model recovery failure rate: #{failed}/#{total} attempts failed"
+          Rails.logger.error(message)
+
+          # Implement your alerting mechanism here
+          # Example: Sentry.capture_message(message, level: :error)
+          # Example: Slack.notify(message, channel: '#alerts')
+        end
+
+        # Sends alert when circuit breaker opens
+        def alert_on_circuit_breaker_open
+          message = 'ReadModelRecoveryJob circuit breaker opened - job execution suspended'
+          Rails.logger.error(message)
+
+          # Implement critical alerting here
+          # Example: PagerDuty.trigger(message)
+        end
+
+        # Sends alert for long stuck models
+        # @param models [Array<ActiveRecord::Base>] The stuck models
+        def alert_on_long_stuck_models(models)
+          message = "#{models.size} read models stuck for > 5 minutes: " \
+                    "#{models.map { |m| "#{m.class.name}##{m.id}" }.join(', ')}"
+          Rails.logger.error(message)
+
+          # Implement critical alerting here
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/middlewares/encryptor.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module Middlewares
+      # PgEventstore middleware for encrypting/decrypting event data.
+      #
+      # @example
+      #   PgEventstore.configure do |config|
+      #     config.middlewares[:encryptor] = Yes::Core::Middlewares::Encryptor.new(key_repository)
+      #   end
+      class Encryptor
+        include PgEventstore::Middleware
+
+        attr_reader :key_repository
+        private :key_repository
+
+        # @param key_repository [#find, #create, #encrypt, #decrypt]
+        def initialize(key_repository)
+          @key_repository = key_repository
+        end
+
+        # @param event [PgEventstore::Event]
+        # @return [PgEventstore::Event]
+        def serialize(event)
+          return event unless event.class.respond_to?(:encryption_schema)
+
+          encryptor = DataEncryptor.new(
+            data: event.data, schema: event.class.encryption_schema, repository: key_repository
+          )
+          encryptor.call
+          event.data = encryptor.encrypted_data
+          event.metadata['encryption'] = encryptor.encryption_metadata
+          event
+        end
+
+        # @param event [PgEventstore::Event]
+        # @return [PgEventstore::Event]
+        def deserialize(event)
+          decrypted_data =
+            DataDecryptor.new(data: event.data, schema: event.metadata['encryption'], repository: key_repository).call
+          event.data = decrypted_data
+          event
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/middlewares/timestamp.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module Middlewares
+      # PgEventstore middleware that adds a created_at timestamp to event metadata
+      # on serialization and parses it back on deserialization.
+      class Timestamp
+        include PgEventstore::Middleware
+
+        # @param event [PgEventstore::Event]
+        # @return [PgEventstore::Event]
+        def serialize(event)
+          event.metadata[:created_at] ||= Time.now.utc
+          event
+        end
+
+        # @param event [PgEventstore::Event]
+        # @return [PgEventstore::Event]
+        def deserialize(event)
+          return event unless event.metadata.key?('created_at')
+
+          event.metadata['created_at'] = Time.zone.parse(event.metadata['created_at'])
+          event
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/middlewares/with_indifferent_access.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module Middlewares
+      # PgEventstore middleware that converts event data and metadata
+      # to HashWithIndifferentAccess, allowing both string and symbol key access.
+      class WithIndifferentAccess
+        include PgEventstore::Middleware
+
+        # @param event [PgEventstore::Event]
+        # @return [PgEventstore::Event]
+        def serialize(event)
+          event.metadata = event.metadata.with_indifferent_access
+          event.data = event.data.with_indifferent_access
+          event
+        end
+        alias deserialize serialize
+      end
+    end
+  end
+end

data/lib/yes/core/models/application_record.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    class ApplicationRecord < ActiveRecord::Base
+      self.abstract_class = true
+    end
+  end
+end

data/lib/yes/core/open_telemetry/otl_span.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module OpenTelemetry
+      # Wraps OpenTelemetry span creation with SQL tracking support.
+      #
+      # @example
+      #   span = OtlSpan.new(otl_data: OtlData.new(span_name: 'MySpan'), otl_tracer: tracer)
+      #   span.otl_span(arg1, arg2) { do_work }
+      class OtlSpan
+        # Configuration struct for OpenTelemetry span data
+        OtlData = Struct.new(:span_name, :span_kind, :span_attributes, :links_extractor, :track_sql) do
+          # @param span_name [String, nil] name of the span
+          # @param span_kind [Symbol] kind of span (:internal, :client, :server, :producer, :consumer)
+          # @param span_attributes [Hash] additional span attributes
+          # @param links_extractor [Proc] extracts OTL context links from arguments
+          # @param track_sql [Boolean] whether to track SQL queries within the span
+          def initialize(span_name: nil, span_kind: :internal, span_attributes: {}, links_extractor: proc { [] },
+                         track_sql: false)
+            super
+          end
+        end
+
+        # @return [OtlData] span configuration
+        attr_reader :otl_data
+
+        # @return [Object] OpenTelemetry tracer instance
+        attr_reader :otl_tracer
+
+        # @param otl_data [OtlData] span configuration
+        # @param otl_tracer [Object] OpenTelemetry tracer instance
+        def initialize(otl_data:, otl_tracer:)
+          @otl_data = otl_data
+          @otl_tracer = otl_tracer
+        end
+
+        # Creates a span and executes the given block within it.
+        #
+        # @param args [Array] positional arguments passed to the links extractor
+        # @param kwargs [Hash] keyword arguments passed to the links extractor
+        # @yield the block to execute within the span
+        # @return [Object] the return value of the block
+        def otl_span(*args, **kwargs, &block)
+          links = otl_links(args, kwargs)
+
+          parent_span = ::OpenTelemetry::Trace.current_span.context.valid? ? ::OpenTelemetry::Trace.current_span : nil
+          root_track_sql = parent_span&.try(:attributes)&.[]('root_track_sql') ||
+                           parent_span&.try(:attributes)&.[]('track_sql')
+
+          otl_tracer.in_span(
+            otl_data.span_name || 'UnknownName',
+            links:,
+            kind: otl_data.span_kind,
+            attributes: {
+              'track_sql' => otl_data.track_sql,
+              'root_track_sql' => root_track_sql || false
+            }.merge(otl_data.span_attributes)
+          ) do
+            next unless block_given?
+            next yield if !root_track_sql && !otl_data.track_sql
+            next yield if parent_span.present? && root_track_sql
+
+            callback = lambda do |sql_event|
+              next if %w[SCHEMA TRANSACTION].include?(sql_event.payload[:name])
+
+              otl_tracer.in_span("SQL #{sql_event.payload[:name]}") do |span|
+                span.set_attribute('db.system', 'postgresql')
+                span.set_attribute('db.statement', sql_event.payload[:sql])
+                span.set_attribute('db.binds', sql_event.payload[:binds].map do |attr|
+                  next { name: attr.name, value: attr.value } if attr.respond_to?(:name) && attr.respond_to?(:value)
+
+                  { name: attr.class.to_s, value: attr }
+                end.to_json)
+                span.set_attribute('db.event_name', sql_event.payload[:name])
+              end
+            end
+            ActiveSupport::Notifications.subscribed(callback, 'sql.active_record', &block)
+          end
+        end
+
+        private
+
+        # Extracts OpenTelemetry links from arguments using the configured links_extractor.
+        #
+        # @param args [Array] positional arguments
+        # @param kwargs [Hash] keyword arguments
+        # @return [Array<OpenTelemetry::Trace::Link>] extracted links
+        def otl_links(args, kwargs)
+          return [] if args.blank? && kwargs.blank?
+          return [] unless (otl_contexts = otl_data.links_extractor.call(*args, **kwargs).presence)
+
+          otl_contexts.filter_map do |_context_name, context_data|
+            next unless context_data['traceparent']
+
+            trace_parent_ctx = ::OpenTelemetry::Trace::Propagation::TraceContext::TraceParent.from_string(
+              context_data['traceparent']
+            )
+            trace_span_ctx = ::OpenTelemetry::Trace::SpanContext.new(
+              trace_id: trace_parent_ctx.trace_id,
+              span_id: trace_parent_ctx.span_id,
+              trace_flags: trace_parent_ctx.flags
+            )
+            ::OpenTelemetry::Trace::Link.new(trace_span_ctx)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/open_telemetry/trackable.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module OpenTelemetry
+      # Mixin for adding OpenTelemetry tracing to classes.
+      #
+      # When OpenTelemetry is not configured (no tracer set), all tracing
+      # operations are no-ops with zero overhead.
+      #
+      # @example
+      #   class MyService
+      #     include Yes::Core::OpenTelemetry::Trackable
+      #
+      #     def process(data)
+      #       # implementation
+      #     end
+      #     otl_trackable :process, OtlSpan::OtlData.new(span_name: 'Process Data')
+      #   end
+      module Trackable
+        extend ActiveSupport::Concern
+
+        module ClassMethods
+          # Decorates a method with OpenTelemetry tracing.
+          #
+          # @param method_name [Symbol] the method to track
+          # @param otl_data [OtlSpan::OtlData] span configuration
+          # @return [Symbol] the tracked method name
+          def otl_trackable(method_name, otl_data = OtlSpan::OtlData.new(span_name: nil))
+            otl_data.span_name ||= name
+
+            instance_module = Module.new do
+              define_method(method_name) do |*args, **kwargs, &blk|
+                return super(*args, **kwargs, &blk) unless singleton_class.otl_tracer
+
+                OtlSpan.new(otl_data:, otl_tracer: singleton_class.otl_tracer).otl_span(*args, **kwargs) do
+                  super(*args, **kwargs, &blk)
+                end
+              end
+            end
+            prepend instance_module
+
+            method_name
+          end
+
+          # @return [Object, nil] the configured OpenTelemetry tracer or nil
+          def otl_tracer
+            Yes::Core.configuration.otl_tracer
+          end
+
+          # @return [OpenTelemetry::Trace::Span, nil] the current span or nil if no tracer
+          def current_span
+            return nil unless otl_tracer
+
+            ::OpenTelemetry::Trace.current_span
+          end
+
+          # @return [OpenTelemetry::Context, nil] the current context or nil if no tracer
+          def current_context
+            return nil unless otl_tracer
+
+            ::OpenTelemetry::Context.current
+          end
+
+          # Executes a block within a new span.
+          #
+          # @param name [String] the span name
+          # @yield the block to execute
+          # @return [Object] the return value of the block
+          def with_otl_span(name, &)
+            return yield unless otl_tracer
+
+            otl_tracer.in_span(name, &)
+          end
+
+          # Propagates the current context to a carrier hash.
+          #
+          # @param ctx_carrier [Hash] the carrier to inject context into
+          # @param service_name [Boolean] whether to include the service name
+          # @return [HashWithIndifferentAccess] the carrier with context
+          def propagate_context(ctx_carrier = {}, service_name: false)
+            ctx_carrier.tap do |carrier|
+              ::OpenTelemetry.propagation.inject(carrier)
+              ctx_carrier[:service] = Rails.application.class.module_parent.name if service_name
+            end.with_indifferent_access
+          end
+
+          # Extracts context from a carrier hash.
+          #
+          # @param carrier [Hash, nil] the carrier to extract from
+          # @return [OpenTelemetry::Context, nil] the extracted context
+          def extract_current_context(carrier)
+            return nil unless carrier&.key?('traceparent')
+
+            ::OpenTelemetry.propagation.extract(carrier)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/payload_store/base.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module PayloadStore
+      # Base class for payload store operations providing shared error handling
+      # and client access.
+      class Base
+        private
+
+        # @param response [Object] the payload store response
+        # @param events [Object] the events associated with the request
+        # @return [Boolean] true if the response was an error
+        def handle_payload_store_error(response, events)
+          return false if response.success?
+
+          msg = "Payload Store error for class: #{self.class.name}"
+          error = Yes::Core::PayloadStore::Errors::ClientError.new(
+            msg, extra: { payload_store_response: response.failure, events: }
+          )
+          Yes::Core::Utils::ErrorNotifier.new.payload_extraction_failed(error)
+
+          true
+        end
+
+        # @return [Object] the configured payload store client
+        def payload_store_client
+          @payload_store_client ||= Yes::Core.configuration.payload_store_client
+        end
+      end
+    end
+  end
+end

data/lib/yes/core/payload_store/errors.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module PayloadStore
+      # Error classes for payload store operations
+      module Errors
+        class MissingClient < Error; end
+        class ClientError < Error; end
+      end
+    end
+  end
+end

data/lib/yes/core/payload_store/lookup.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Yes
+  module Core
+    module PayloadStore
+      # Resolves payload store references in events by looking up stored values.
+      class Lookup < Base
+        # @param event [PgEventstore::Event] the event with potential payload store references
+        # @return [Hash] resolved key-value pairs from the payload store
+        def call(event)
+          return {} unless event.respond_to?(:ps_fields_with_values)
+
+          keys = event.ps_fields_with_values
+          return {} if keys.blank?
+
+          raise Yes::Core::PayloadStore::Errors::MissingClient unless payload_store_client
+
+          ps_response = payload_store_client.get_payloads(keys.values)
+
+          unless ps_response.success?
+            handle_payload_store_error(ps_response, event)
+            return {}
+          end
+
+          resolved_payloads(ps_response, keys)
+        end
+
+        private
+
+        # @param response [Object] the successful payload store response
+        # @param keys [Hash] the original key mappings
+        # @return [Hash] resolved payloads mapped back to their original keys
+        def resolved_payloads(response, keys)
+          response.value!.each_with_object({}) do |resp, ps_attr_hash|
+            key = keys.key(resp.attributes[:key])
+            next unless key
+
+            ps_attr_hash[key] = resp.attributes[:value]
+          end
+        end
+      end
+    end
+  end
+end