ez_logs_agent 0.1.0

data/lib/ez_logs_agent/capturers/database_capturer.rb

@@ -0,0 +1,467 @@
+# frozen_string_literal: true
+
+module EzLogsAgent
+  module Capturers
+    # Captures database operations via ActiveRecord model lifecycle callbacks.
+    #
+    # This capturer:
+    # - Installs after_create, after_update, after_destroy callbacks on ActiveRecord::Base
+    # - Captures model class, record id, and operation type
+    # - Extracts resource_ids from the model instance
+    # - For updates, extracts curated business-relevant change context
+    # - Preserves correlation_id from current context
+    # - Never crashes the host application (fail-open)
+    # - Respects capture_database configuration flag
+    #
+    # == What This Capturer Does NOT Do
+    #
+    # - Parse SQL queries
+    # - Dump full attribute diffs
+    # - Include sensitive data
+    # - Guess actors
+    # - Act as an audit log
+    #
+    # == Event Shape
+    #
+    # Produces events with:
+    # - source_type: :database_callback
+    # - source_data: { model_class: "User", operation: "create|update|destroy" }
+    # - outcome: :success
+    # - correlation_id: EzLogsAgent::Correlation.current (if present)
+    # - resource_ids: [{ resource_type: "User", resource_id: "123" }]
+    # - context: { changes: [{ attribute: "status", from: "pending", to: "shipped" }, ...] } (updates only, if meaningful)
+    #
+    class DatabaseCapturer
+      # Attributes to always ignore when detecting business changes
+      IGNORED_ATTRIBUTES = %w[
+        id
+        created_at
+        updated_at
+        lock_version
+        encrypted_password
+        reset_password_token
+        reset_password_sent_at
+        remember_created_at
+        confirmation_token
+        confirmed_at
+        confirmation_sent_at
+        unconfirmed_email
+        unlock_token
+        locked_at
+        sign_in_count
+        current_sign_in_at
+        last_sign_in_at
+        current_sign_in_ip
+        last_sign_in_ip
+      ].freeze
+
+      # Foreign key changes are now captured because they represent meaningful
+      # relationship changes (e.g., profile_id, user_id).
+      # Previously we filtered them out, but this loses important context.
+      # FOREIGN_KEY_PATTERN = /_id\z/  # Removed January 2026
+
+      # Patterns for sensitive data to ignore
+      SENSITIVE_PATTERNS = %w[
+        password
+        token
+        secret
+        api_key
+        credit_card
+        ssn
+        social_security
+        encrypted
+      ].freeze
+
+
+      @installed = false
+      @callbacks_registered = false
+
+      class << self
+        # Installs ActiveRecord lifecycle callbacks for database capture.
+        #
+        # This method is idempotent and can be called multiple times safely.
+        # Only installs if ActiveRecord is present.
+        #
+        # @return [void]
+        def install
+          return unless defined?(ActiveRecord::Base)
+          return if @installed
+
+          # Only register callbacks once per Ruby process
+          unless @callbacks_registered
+            ActiveRecord::Base.class_eval do
+              after_create { |model| EzLogsAgent::Capturers::DatabaseCapturer.handle_create(model) }
+              after_update { |model| EzLogsAgent::Capturers::DatabaseCapturer.handle_update(model) }
+              after_destroy { |model| EzLogsAgent::Capturers::DatabaseCapturer.handle_destroy(model) }
+            end
+            @callbacks_registered = true
+          end
+
+          @installed = true
+          EzLogsAgent::Logger.debug("[DatabaseCapturer] Installed")
+        rescue StandardError => e
+          EzLogsAgent::Logger.error("[DatabaseCapturer] Installation failed: #{e.class} - #{e.message}")
+        end
+
+        # Handles after_create callback
+        #
+        # @param model [ActiveRecord::Base] The created model instance
+        # @return [void]
+        def handle_create(model)
+          return unless capture_enabled?
+
+          context = extract_initial_attributes(model) || {}
+          context[:display_name] = resolve_display_name(model)
+          capture_event(model, "create", context: context.presence)
+        rescue StandardError => e
+          EzLogsAgent::Logger.error("[DatabaseCapturer] handle_create failed: #{e.class} - #{e.message}")
+        end
+
+        # Handles after_update callback
+        #
+        # @param model [ActiveRecord::Base] The updated model instance
+        # @return [void]
+        def handle_update(model)
+          return unless capture_enabled?
+
+          context = extract_change_context(model) || {}
+          context[:display_name] = resolve_display_name(model)
+          capture_event(model, "update", context: context.presence)
+        rescue StandardError => e
+          EzLogsAgent::Logger.error("[DatabaseCapturer] handle_update failed: #{e.class} - #{e.message}")
+        end
+
+        # Handles after_destroy callback
+        #
+        # @param model [ActiveRecord::Base] The destroyed model instance
+        # @return [void]
+        def handle_destroy(model)
+          return unless capture_enabled?
+
+          context = { display_name: resolve_display_name(model) }
+          capture_event(model, "destroy", context: context.presence)
+        rescue StandardError => e
+          EzLogsAgent::Logger.error("[DatabaseCapturer] handle_destroy failed: #{e.class} - #{e.message}")
+        end
+
+        private
+
+        # Checks if database capture is enabled
+        #
+        # @return [Boolean]
+        def capture_enabled?
+          EzLogsAgent.configuration.capture_database
+        rescue StandardError
+          false
+        end
+
+        # Checks if the model's table is in the excluded_tables list
+        # Uses all_excluded_tables which combines defaults with user-configured
+        #
+        # @param model [ActiveRecord::Base] The model instance
+        # @return [Boolean]
+        def table_excluded?(model)
+          return false unless model.class.respond_to?(:table_name)
+
+          EzLogsAgent.configuration.all_excluded_tables.include?(model.class.table_name)
+        rescue StandardError
+          false
+        end
+
+        # Captures a database event and pushes to buffer
+        #
+        # @param model [ActiveRecord::Base] The model instance
+        # @param operation [String] The operation type ("create", "update", "destroy")
+        # @param context [Hash, nil] Optional context with change information
+        # @return [void]
+        def capture_event(model, operation, context: nil)
+          return if table_excluded?(model)
+
+          event = EzLogsAgent::EventBuilder.build(
+            source_type: :database_callback,
+            source_data: {
+              model_class: model.class.name,
+              operation: operation
+            },
+            outcome: :success,
+            correlation_id: EzLogsAgent::Correlation.current,
+            resource_ids: extract_resource_ids(model),
+            context: context,
+            duration_ms: nil
+          )
+
+          EzLogsAgent::Buffer.push(event)
+        rescue StandardError => e
+          EzLogsAgent::Logger.error("[DatabaseCapturer] capture_event failed: #{e.class} - #{e.message}")
+        end
+
+        # Extracts resource_ids from model
+        #
+        # @param model [ActiveRecord::Base] The model instance
+        # @return [Array<Hash>] Array with single resource identifier
+        def extract_resource_ids(model)
+          [
+            {
+              resource_type: model.class.name,
+              resource_id: model.id.to_s
+            }
+          ]
+        rescue StandardError
+          []
+        end
+
+        # Resolves a human-readable display name for a model instance
+        #
+        # Uses configuration if provided, otherwise falls back to common patterns:
+        # 1. Custom field from config.display_name_for[ModelClass]
+        # 2. model.name (if responds)
+        # 3. model.title (if responds)
+        # 4. model.number (if responds)
+        #
+        # Returns nil if no meaningful name found. The frontend can decide
+        # to show the resource ID as a fallback if needed.
+        #
+        # IMPORTANT: This method only reads attributes already loaded in memory.
+        # It does NOT trigger any database queries.
+        # Configured fields should be direct attributes, not associations.
+        #
+        # @param model [ActiveRecord::Base] The model instance
+        # @return [String, nil] The display name, or nil if no meaningful name found
+        def resolve_display_name(model)
+          # Check for configured custom field
+          display_name_config = EzLogsAgent.configuration.display_name_for || {}
+          custom_field = display_name_config[model.class.name]
+
+          if custom_field && model.respond_to?(custom_field)
+            value = model.public_send(custom_field)
+            return value.to_s if value.present?
+          end
+
+          # Fallback chain: name → title → number
+          if model.respond_to?(:name) && model.name.present?
+            return model.name.to_s
+          end
+
+          if model.respond_to?(:title) && model.title.present?
+            return model.title.to_s
+          end
+
+          if model.respond_to?(:number) && model.number.present?
+            return model.number.to_s
+          end
+
+          # No meaningful name found - return nil
+          # Frontend can show resource ID as fallback if needed
+          nil
+        rescue StandardError => e
+          EzLogsAgent::Logger.error("[DatabaseCapturer] resolve_display_name failed: #{e.class} - #{e.message}")
+          nil
+        end
+
+        # Extracts curated business change context from model's saved_changes
+        #
+        # Rules:
+        # - Only for updates
+        # - Pick meaningful attributes (ignore technical fields)
+        # - Ignore foreign keys
+        # - Ignore sensitive data
+        # - Only scalar values (String, Integer, Float, Boolean, Symbol, NilClass)
+        # - Value must have actually changed (from != to, not both nil)
+        #
+        # @param model [ActiveRecord::Base] The updated model instance
+        # @return [Hash, nil] Context hash with change, or nil if no meaningful change
+        def extract_change_context(model)
+          return nil unless model.respond_to?(:saved_changes)
+
+          changes = model.saved_changes
+          return nil if changes.nil? || changes.empty?
+
+          # Find meaningful changes
+          meaningful_changes = filter_meaningful_changes(changes)
+          return nil if meaningful_changes.empty?
+
+          # Build context with all meaningful changes
+          build_change_context(meaningful_changes)
+        rescue StandardError => e
+          EzLogsAgent::Logger.error("[DatabaseCapturer] extract_change_context failed: #{e.class} - #{e.message}")
+          nil
+        end
+
+        # Extracts initial attributes from a newly created model
+        #
+        # Uses the same filtering rules as update changes:
+        # - Only meaningful attributes (not technical/ignored)
+        # - Only scalar values
+        # - Skip nil values (no point recording "attribute: nil")
+        # - Skip foreign keys and sensitive data
+        #
+        # @param model [ActiveRecord::Base] The created model instance
+        # @return [Hash, nil] Context hash with initial_attributes, or nil if none
+        def extract_initial_attributes(model)
+          return nil unless model.respond_to?(:attributes)
+
+          attributes = model.attributes
+          return nil if attributes.nil? || attributes.empty?
+
+          # Filter to meaningful, non-nil scalar attributes
+          meaningful_attrs = attributes.select do |attribute, value|
+            meaningful_attribute?(attribute) &&
+              scalar?(value) &&
+              !value.nil?
+          end
+
+          return nil if meaningful_attrs.empty?
+
+          # Format values for JSON
+          formatted_attrs = meaningful_attrs.transform_values { |v| format_value_for_json(v) }
+
+          { initial_attributes: formatted_attrs }
+        rescue StandardError => e
+          EzLogsAgent::Logger.error("[DatabaseCapturer] extract_initial_attributes failed: #{e.class} - #{e.message}")
+          nil
+        end
+
+        # Filters changes to only meaningful business attributes
+        #
+        # @param changes [Hash] The saved_changes hash
+        # @return [Array<Array>] Array of [attribute, [from, to]] pairs
+        def filter_meaningful_changes(changes)
+          changes.select do |attribute, (from, to)|
+            meaningful_attribute?(attribute) &&
+              scalar_values?(from, to) &&
+              values_actually_changed?(from, to)
+          end.to_a
+        end
+
+        # Checks if an attribute is meaningful (not technical/ignored)
+        #
+        # @param attribute [String] The attribute name
+        # @return [Boolean]
+        def meaningful_attribute?(attribute)
+          attr_str = attribute.to_s
+
+          # Skip explicitly ignored attributes
+          return false if IGNORED_ATTRIBUTES.include?(attr_str)
+
+          # Foreign keys (_id) are now captured because they represent meaningful
+          # relationship changes (e.g., assigned_to_id changing from user A to user B)
+          # Previously filtered via FOREIGN_KEY_PATTERN - removed January 2026
+
+          # Skip sensitive data
+          return false if sensitive_attribute?(attr_str)
+
+          true
+        end
+
+        # Checks if attribute name contains sensitive patterns
+        #
+        # @param attribute [String] The attribute name
+        # @return [Boolean]
+        def sensitive_attribute?(attribute)
+          attr_lower = attribute.downcase
+          SENSITIVE_PATTERNS.any? { |pattern| attr_lower.include?(pattern) }
+        end
+
+        # Checks if both values are scalar types
+        #
+        # @param from [Object] The old value
+        # @param to [Object] The new value
+        # @return [Boolean]
+        def scalar_values?(from, to)
+          scalar?(from) && scalar?(to)
+        end
+
+        # Checks if a value is a scalar type (simple, serializable values)
+        #
+        # Includes Date/Time types which are meaningful business values
+        # (e.g., discarded_at for soft deletes, published_at, expires_at)
+        #
+        # Includes BigDecimal which Rails uses for decimal columns
+        # (e.g., prices, percentages, rates)
+        #
+        # Includes arrays of scalar values (e.g., PostgreSQL array columns
+        # like email lists)
+        #
+        # @param value [Object] The value to check
+        # @return [Boolean]
+        def scalar?(value)
+          return true if value.nil? ||
+            value.is_a?(String) ||
+            value.is_a?(Integer) ||
+            value.is_a?(Float) ||
+            value.is_a?(BigDecimal) ||
+            value.is_a?(TrueClass) ||
+            value.is_a?(FalseClass) ||
+            value.is_a?(Symbol) ||
+            value.is_a?(Date) ||
+            value.is_a?(Time)
+          # Note: DateTime inherits from Date, so Date check covers it
+
+          # Arrays of scalars are also allowed (e.g., email arrays)
+          return value.all? { |v| scalar?(v) } if value.is_a?(Array)
+
+          false
+        end
+
+        # Checks if values actually changed (not both nil)
+        #
+        # @param from [Object] The old value
+        # @param to [Object] The new value
+        # @return [Boolean]
+        def values_actually_changed?(from, to)
+          # Both nil means no real change
+          return false if from.nil? && to.nil?
+
+          # Values must be different
+          from != to
+        end
+
+        # Builds context hash with all meaningful changes
+        #
+        # Captures ALL meaningful changes for better visibility:
+        # - Shows what actually changed in the UI
+        # - Enables better event descriptions
+        # - Still excludes sensitive/technical fields
+        # - Formats Date/Time values as ISO strings for JSON serialization
+        #
+        # @param meaningful_changes [Array<Array>] Array of [attribute, [from, to]] pairs
+        # @return [Hash] Context hash with changes array
+        def build_change_context(meaningful_changes)
+          changes = meaningful_changes.map do |attribute, (from, to)|
+            {
+              attribute: attribute.to_s,
+              from: format_value_for_json(from),
+              to: format_value_for_json(to)
+            }
+          end
+
+          { changes: changes }
+        end
+
+        # Formats a value for JSON serialization
+        #
+        # Date/Time values are converted to ISO 8601 strings
+        # BigDecimal values are converted to floats (JSON doesn't support BigDecimal)
+        # Arrays are recursively formatted
+        # Other values pass through unchanged
+        #
+        # @param value [Object] The value to format
+        # @return [Object] The formatted value
+        def format_value_for_json(value)
+          case value
+          when Time, DateTime
+            value.iso8601
+          when Date
+            value.to_s
+          when BigDecimal
+            value.to_f
+          when Array
+            value.map { |v| format_value_for_json(v) }
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ez_logs_agent/capturers/job_capturer.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+module EzLogsAgent
+  module Capturers
+    class JobCapturer
+      # Sidekiq client middleware that propagates correlation_id
+      # from the current request context into enqueued job payloads.
+      #
+      # This middleware runs at enqueue-time (when a job is scheduled),
+      # NOT at execution-time.
+      #
+      # == Behavior
+      #
+      # - Reads EzLogsAgent::Correlation.current
+      # - If present, injects it into job["correlation_id"]
+      # - Never overwrites existing job["correlation_id"]
+      # - Never raises exceptions
+      # - Always yields to next middleware
+      #
+      # == Registration
+      #
+      # Note: This middleware is automatically registered by Railtie.
+      # Users do NOT need to manually configure it.
+      #
+      # For manual setup (if not using Rails), add to your Sidekiq initializer:
+      #
+      #   Sidekiq.configure_client do |config|
+      #     config.client_middleware do |chain|
+      #       chain.add EzLogsAgent::Capturers::JobCapturer::ClientMiddleware
+      #     end
+      #   end
+      #
+      class ClientMiddleware
+        # Sidekiq client middleware hook.
+        #
+        # @param worker_class [Class] The worker class being enqueued
+        # @param job [Hash] The Sidekiq job payload (mutable)
+        # @param queue [String] The queue name
+        # @param redis_pool [ConnectionPool] Sidekiq's Redis connection pool
+        # @yield Passes control to the next middleware in the chain
+        # @return [void]
+        def call(worker_class, job, queue, redis_pool)
+          # Defensive: ensure job is a Hash
+          return yield unless job.is_a?(Hash)
+
+          begin
+            # Read current correlation_id from context
+            correlation_id = EzLogsAgent::Correlation.current
+
+            # Only inject if:
+            # 1. correlation_id is present and not empty
+            # 2. job doesn't already have one
+            if correlation_id && correlation_id.respond_to?(:empty?) && !correlation_id.empty? && !job.key?("correlation_id")
+              job["correlation_id"] = correlation_id
+            end
+          rescue StandardError => e
+            # Never crash the host application during correlation injection
+            # Log error, then continue
+            EzLogsAgent::Logger.error("[JobCapturer::ClientMiddleware] Error during correlation injection: #{e.class} - #{e.message}")
+          end
+
+          # Always yield to next middleware
+          yield
+        end
+      end
+
+      # Sidekiq server middleware that captures background job execution
+      # as background_job events.
+      #
+      # This middleware runs at execution-time (when the job runs),
+      # NOT at enqueue-time.
+      #
+      # == Behavior
+      #
+      # - Restores correlation_id from job["correlation_id"] (or generates new one)
+      # - Measures job execution duration
+      # - Captures success or failure outcome
+      # - Re-raises exceptions after capturing failure
+      # - Always clears correlation_id in ensure block
+      # - Respects capture_jobs configuration flag
+      #
+      # == Registration
+      #
+      # Note: This middleware is automatically registered by Railtie.
+      # Users do NOT need to manually configure it.
+      #
+      # For manual setup (if not using Rails), add to your Sidekiq initializer:
+      #
+      #   Sidekiq.configure_server do |config|
+      #     config.server_middleware do |chain|
+      #       chain.add EzLogsAgent::Capturers::JobCapturer::ServerMiddleware
+      #     end
+      #   end
+      #
+      class ServerMiddleware
+        # Sidekiq server middleware hook.
+        #
+        # @param worker [Object] The worker instance executing the job
+        # @param job [Hash] The Sidekiq job payload
+        # @param queue [String] The queue name
+        # @yield Executes the job
+        # @return [void]
+        def call(worker, job, queue)
+          # Skip capture if disabled
+          return yield unless EzLogsAgent.configuration.capture_jobs
+
+          # Skip excluded job classes (health checks, infrastructure jobs).
+          # Match against the underlying ActiveJob class too, not just the wrapper.
+          if excluded_job_class?(worker, job)
+            EzLogsAgent::Logger.debug("[JobCapturer::ServerMiddleware] Skipping capture (excluded job class: #{resolve_job_class(worker, job)})")
+            return yield
+          end
+
+          # Restore correlation_id from job payload (or generate new one)
+          correlation_id = job["correlation_id"] || EzLogsAgent::Correlation.generate
+          EzLogsAgent::Correlation.current = correlation_id
+
+          # Measure execution time
+          start_time = Time.now
+          result = yield
+          duration_ms = ((Time.now - start_time) * 1000).to_i
+
+          # Capture success event
+          capture_success(worker, job, queue, correlation_id, duration_ms, start_time)
+
+          result
+        rescue StandardError => error
+          # Capture failure event, then re-raise
+          capture_failure(worker, job, queue, correlation_id, error, start_time)
+          raise
+        ensure
+          # Always clear correlation_id
+          EzLogsAgent::Correlation.clear
+        end
+
+        private
+
+        # Captures successful job execution.
+        #
+        # @param worker [Object] The worker instance
+        # @param job [Hash] The Sidekiq job payload
+        # @param queue [String] The queue name
+        # @param correlation_id [String] The correlation ID
+        # @param duration_ms [Integer] Job execution duration in milliseconds
+        # @param start_time [Time] Job start time
+        # @return [void]
+        def capture_success(worker, job, queue, correlation_id, duration_ms, start_time)
+          event = EzLogsAgent::EventBuilder.build(
+            source_type: :background_job,
+            source_data: extract_job_data(worker, job, queue),
+            outcome: :success,
+            correlation_id: correlation_id,
+            duration_ms: duration_ms,
+            timestamp: start_time
+          )
+
+          EzLogsAgent::Buffer.push(event)
+        rescue StandardError => e
+          # Defensive: never crash the host application during event capture
+          EzLogsAgent::Logger.error("[JobCapturer::ServerMiddleware] Failed to capture success event: #{e.class} - #{e.message}")
+        end
+
+        # Captures failed job execution.
+        #
+        # @param worker [Object] The worker instance
+        # @param job [Hash] The Sidekiq job payload
+        # @param queue [String] The queue name
+        # @param correlation_id [String] The correlation ID
+        # @param error [StandardError] The error that caused the failure
+        # @param start_time [Time] Job start time
+        # @return [void]
+        def capture_failure(worker, job, queue, correlation_id, error, start_time)
+          event = EzLogsAgent::EventBuilder.build(
+            source_type: :background_job,
+            source_data: extract_job_data(worker, job, queue),
+            outcome: :failure,
+            correlation_id: correlation_id,
+            error_message: "#{error.class}: #{error.message}",
+            timestamp: start_time
+          )
+
+          EzLogsAgent::Buffer.push(event)
+        rescue StandardError => e
+          # Defensive: never crash the host application during event capture
+          EzLogsAgent::Logger.error("[JobCapturer::ServerMiddleware] Failed to capture failure event: #{e.class} - #{e.message}")
+        end
+
+        # Extracts relevant job data for event source_data.
+        #
+        # When the worker is the ActiveJob+Sidekiq wrapper, we surface the
+        # underlying ActiveJob class name (e.g., "ChargeCardJob") rather
+        # than the wrapper class. This mirrors how Sidekiq itself resolves
+        # the display class — see Sidekiq::JobLogger and Sidekiq::JobUtil:
+        #
+        #   job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"]
+        #
+        # @param worker [Object] The worker instance
+        # @param job [Hash] The Sidekiq job payload
+        # @param queue [String] The queue name
+        # @return [Hash] Job metadata for source_data
+        def extract_job_data(worker, job, queue)
+          {
+            job_class: resolve_job_class(worker, job),
+            job_id: job["jid"],
+            queue: queue,
+            retry_count: job["retry_count"]
+          }.compact
+        end
+
+        # Resolves the human-meaningful job class name, unwrapping the
+        # ActiveJob+Sidekiq adapter when present.
+        #
+        # @param worker [Object] The worker instance Sidekiq is running
+        # @param job [Hash] The Sidekiq job payload
+        # @return [String] The class name to record on the event
+        def resolve_job_class(worker, job)
+          job["display_class"] || job["wrapped"] || worker.class.name
+        end
+
+        # Checks if worker class is in the excluded list.
+        #
+        # Exclusion is matched against the same name that lands in
+        # source_data, so users can list either the Sidekiq worker class
+        # or the underlying ActiveJob class — whichever they actually own.
+        #
+        # @param worker [Object] The worker instance
+        # @param job [Hash] The Sidekiq job payload (optional; defaults to {})
+        # @return [Boolean] true if excluded, false otherwise
+        def excluded_job_class?(worker, job = {})
+          excluded = EzLogsAgent.configuration.all_excluded_job_classes
+          [worker.class.name, job["wrapped"], job["display_class"]].compact.any? { |name| excluded.include?(name) }
+        rescue StandardError
+          false
+        end
+      end
+    end
+  end
+end