ez_logs_agent 0.1.10 → 0.2.1

data/lib/ez_logs_agent/capturers/database_capturer.rb

@@ -60,37 +60,9 @@ module EzLogsAgent
       # Previously we filtered them out, but this loses important context.
       # FOREIGN_KEY_PATTERN = /_id\z/  # Removed January 2026
 
-      # Patterns for sensitive data to ignore.
-      #
-      # The first source of truth is `record.class.encrypted_attributes`
-      # (Rails 7+ `encrypts :foo` declaration) — see encrypted_attribute?.
-      # If the host app encrypted it, we never capture it.
-      #
-      # This list is the secondary defense: column names that frequently
-      # carry sensitive material even when the host app didn't declare
-      # `encrypts` (legacy code, manual hashing, externally-generated
-      # material). Matching is substring + case-insensitive.
-      SENSITIVE_PATTERNS = %w[
-        password
-        token
-        secret
-        api_key
-        credit_card
-        ssn
-        social_security
-        encrypted
-        private_key
-        public_key
-        signing_key
-        pem
-        cipher
-        nonce
-        salt
-        digest
-        signature
-        hmac
-      ].freeze
-
+      # Sensitive-attribute name pattern denylist (secondary defense after
+      # `encrypts :foo` introspection) lives in EzLogsAgent::SensitivePatterns —
+      # see sensitive_attribute? below.
 
       @installed = false
       @callbacks_registered = false
@@ -106,6 +78,33 @@ module EzLogsAgent
           return unless defined?(ActiveRecord::Base)
           return if @installed
 
+          # Memoize config values that the per-row hot path reads on
+          # every callback. Without this, each captured create / update
+          # / destroy pays a method dispatch into the configuration
+          # object (`EzLogsAgent.configuration.capture_database`) plus
+          # an `all_excluded_tables.include?` Hash dispatch. On a request
+          # that touches dozens of rows the overhead is real.
+          # Runtime mutations require uninstall! + install to take effect
+          # (acceptable — nobody flips capture_database at runtime).
+          @capture_enabled =
+            begin
+              EzLogsAgent.configuration.capture_database
+            rescue StandardError
+              false
+            end
+          @excluded_tables =
+            begin
+              EzLogsAgent.configuration.all_excluded_tables.dup.freeze
+            rescue StandardError
+              [].freeze
+            end
+          @display_name_for =
+            begin
+              (EzLogsAgent.configuration.display_name_for || {}).dup.freeze
+            rescue StandardError
+              {}.freeze
+            end
+
           # Only register callbacks once per Ruby process
           unless @callbacks_registered
             ActiveRecord::Base.class_eval do
@@ -165,24 +164,24 @@ module EzLogsAgent
 
         private
 
-        # Checks if database capture is enabled
+        # Checks if database capture is enabled. Reads the memoized
+        # value set at install time (no config-object dispatch on the
+        # hot path).
         #
         # @return [Boolean]
         def capture_enabled?
-          EzLogsAgent.configuration.capture_database
-        rescue StandardError
-          false
+          @capture_enabled
         end
 
-        # Checks if the model's table is in the excluded_tables list
-        # Uses all_excluded_tables which combines defaults with user-configured
+        # Checks if the model's table is in the excluded_tables list.
+        # Reads the memoized list set at install time.
         #
         # @param model [ActiveRecord::Base] The model instance
         # @return [Boolean]
         def table_excluded?(model)
           return false unless model.class.respond_to?(:table_name)
 
-          EzLogsAgent.configuration.all_excluded_tables.include?(model.class.table_name)
+          @excluded_tables.include?(model.class.table_name)
         rescue StandardError
           false
         end
@@ -247,9 +246,8 @@ module EzLogsAgent
         # @param model [ActiveRecord::Base] The model instance
         # @return [String, nil] The display name, or nil if no meaningful name found
         def resolve_display_name(model)
-          # Check for configured custom field
-          display_name_config = EzLogsAgent.configuration.display_name_for || {}
-          custom_field = display_name_config[model.class.name]
+          # Check for configured custom field (memoized list, see install).
+          custom_field = @display_name_for[model.class.name]
 
           if custom_field && model.respond_to?(custom_field)
             value = model.public_send(custom_field)
@@ -386,35 +384,25 @@ module EzLogsAgent
         end
 
         # Checks whether the host app declared `encrypts :<attribute>` on
-        # this model's class. Available since Rails 7.0 via
-        # ActiveRecord::Encryption::EncryptableRecord#encrypted_attributes.
-        #
-        # Safe across host Rails versions: returns false if the API isn't
-        # present (older Rails, non-AR records).
+        # this model's class. Delegates to EncryptedAttributes (single
+        # source of truth shared with BulkDatabaseCapturer, which only has
+        # the class — no instance — for bulk operations).
         #
         # @param attribute [String] The attribute name (already to_s'd)
         # @param model [ActiveRecord::Base] The model instance
         # @return [Boolean]
         def encrypted_attribute?(attribute, model)
-          klass = model.class
-          return false unless klass.respond_to?(:encrypted_attributes)
-
-          encrypted = klass.encrypted_attributes
-          return false if encrypted.nil? || encrypted.empty?
-
-          encrypted.map(&:to_s).include?(attribute)
-        rescue StandardError
-          false
+          EzLogsAgent::EncryptedAttributes.attribute?(model.class, attribute)
         end
 
         # Checks if attribute name contains sensitive patterns.
-        # Secondary check — see SENSITIVE_PATTERNS comment.
+        # Delegates to SensitivePatterns (single source of truth shared
+        # with Sanitizer and BulkDatabaseCapturer).
         #
         # @param attribute [String] The attribute name
         # @return [Boolean]
         def sensitive_attribute?(attribute)
-          attr_lower = attribute.downcase
-          SENSITIVE_PATTERNS.any? { |pattern| attr_lower.include?(pattern) }
+          EzLogsAgent::SensitivePatterns.match?(attribute)
         end
 
         # Checks if both values are scalar types

data/lib/ez_logs_agent/encrypted_attributes.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module EzLogsAgent
+  # Primary defense against capturing encrypted columns: read the host
+  # app's `encrypts :foo` declarations (Rails 7+ ActiveRecord::Encryption)
+  # and drop those attributes from anywhere we'd ship them on the wire.
+  #
+  # Two callers:
+  # - DatabaseCapturer (per-record callbacks) — has a model INSTANCE,
+  #   used to also work from `record.class`.
+  # - BulkDatabaseCapturer (AS::Notifications path) — has only the model
+  #   CLASS (the bulk SQL never instantiated a record). So this module
+  #   takes a class, not an instance — both call sites converge.
+  #
+  # The Rails API is `ModelClass.encrypted_attributes` (Symbol array).
+  # Available since Rails 7.0; older Rails or non-AR classes return false
+  # from this module, which is the fail-open default for the encrypts
+  # check. The pattern-based fallback in SensitivePatterns is the second
+  # layer of defense for hosts that don't (or can't) declare encrypts.
+  module EncryptedAttributes
+    module_function
+
+    # @param model_class [Class, nil] The AR class
+    # @param attribute [String, Symbol, nil] The attribute name
+    # @return [Boolean] true iff the host app declared `encrypts :<attribute>`
+    #   on `model_class` (or an ancestor). False on any error, missing API,
+    #   or empty list — see comment about fail-open semantics above.
+    def attribute?(model_class, attribute)
+      return false if model_class.nil? || attribute.nil?
+      return false unless model_class.respond_to?(:encrypted_attributes)
+
+      declared = model_class.encrypted_attributes
+      return false if declared.nil? || declared.empty?
+
+      attribute_str = attribute.to_s
+      declared.any? { |declared_attr| declared_attr.to_s == attribute_str }
+    rescue StandardError
+      # Same rescue policy as the previous inline check in DatabaseCapturer:
+      # if introspection raises (host app monkey-patched the API, weird
+      # AR class hierarchy), fall through to the pattern-based fallback
+      # rather than crash the capture path.
+      false
+    end
+  end
+end

data/lib/ez_logs_agent/event_builder.rb

@@ -24,7 +24,10 @@ module EzLogsAgent
     SENSITIVE_KEYS = %w[password token secret api_key credit_card].freeze
 
     # Valid source types
-    VALID_SOURCE_TYPES = %w[http_request background_job database_callback].freeze
+    # bulk_database covers AR bulk ops that bypass per-row callbacks
+    # (delete_all, update_all, insert_all, upsert_all) captured via
+    # ActiveSupport::Notifications. See Capturers::BulkDatabaseCapturer.
+    VALID_SOURCE_TYPES = %w[http_request background_job database_callback bulk_database].freeze
 
     # Valid outcome values
     VALID_OUTCOMES = %w[success failure].freeze

data/lib/ez_logs_agent/railtie.rb

@@ -218,10 +218,13 @@ module EzLogsAgent
       EzLogsAgent::Logger.error("[Railtie] Failed to install ActiveJob capturer: #{e.class} - #{e.message}")
     end
 
-    # Install Database capturer
+    # Install Database capturers (per-row + bulk).
     #
-    # Database capturer installs ActiveRecord lifecycle callbacks
-    # (after_create, after_update, after_destroy) for all models.
+    # Two capturers, one switch (`capture_database`):
+    # - DatabaseCapturer: per-row CRUD via after_create / _update / _destroy.
+    # - BulkDatabaseCapturer: bulk SQL via ActiveSupport::Notifications
+    #   ("sql.active_record"), narrowly filtered to delete_all / update_all /
+    #   insert_all / upsert_all. Catches what callbacks can't see.
     #
     # @return [void]
     def self.install_database_capturer
@@ -240,8 +243,9 @@ module EzLogsAgent
       return if @database_capturer_installed
 
       EzLogsAgent::Capturers::DatabaseCapturer.install
+      EzLogsAgent::Capturers::BulkDatabaseCapturer.install
       @database_capturer_installed = true
-      EzLogsAgent::Logger.debug("[Railtie] Database capture installed")
+      EzLogsAgent::Logger.debug("[Railtie] Database capture installed (per-row + bulk)")
     rescue StandardError => e
       EzLogsAgent::Logger.error("[Railtie] Failed to install database capturer: #{e.class} - #{e.message}")
     end

data/lib/ez_logs_agent/sanitizer.rb

@@ -21,18 +21,11 @@ module EzLogsAgent
   # The module is pure (no I/O, no state), so it's safe to call from
   # any thread.
   module Sanitizer
-    # Default sensitive-key patterns. Matched case-insensitively as
-    # SUBSTRINGS of the key, so `customer_password` matches `password`.
-    SENSITIVE_PATTERNS = %w[
-      password passwd pwd
-      token access_token refresh_token api_token auth_token
-      secret api_secret client_secret
-      api_key apikey private_key privatekey secret_key secretkey
-      credential auth authorization
-      encrypted encrypted_data
-      ssn social_security
-      credit_card card_number cvv cvc
-    ].freeze
+    # Sensitive-key pattern list. Delegates to SensitivePatterns (single
+    # source of truth shared with DatabaseCapturer / BulkDatabaseCapturer).
+    # Kept as a constant alias for backwards compatibility — code that
+    # used `Sanitizer::SENSITIVE_PATTERNS` continues to work.
+    SENSITIVE_PATTERNS = EzLogsAgent::SensitivePatterns::PATTERNS
 
     # Hard ceiling for nested object recursion. Deeper structures
     # collapse to the literal string "[Object]".
@@ -79,18 +72,13 @@ module EzLogsAgent
 
       # Check whether a key matches a sensitive pattern. Public so the
       # HTTP middleware can short-circuit early on identical keys.
+      # Delegates to SensitivePatterns (single source of truth — also
+      # consulted by DatabaseCapturer and BulkDatabaseCapturer).
       #
       # @param key [String, Symbol]
       # @return [Boolean]
       def sensitive_key?(key)
-        key_lower = key.to_s.downcase
-        return true if SENSITIVE_PATTERNS.any? { |pattern| key_lower.include?(pattern) }
-
-        user_patterns = EzLogsAgent.configuration.excluded_graphql_variable_keys || []
-        user_patterns.any? { |pattern| key_lower.include?(pattern.to_s.downcase) }
-      rescue
-        # Defensive: when in doubt, treat as sensitive.
-        true
+        EzLogsAgent::SensitivePatterns.match?(key)
       end
 
       private

data/lib/ez_logs_agent/sensitive_patterns.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module EzLogsAgent
+  # Single source of truth for the agent's sensitive-key denylist. Used by
+  # every capture path that needs to mask a value based on its column /
+  # parameter / argument name (Sanitizer for HTTP params + job args,
+  # DatabaseCapturer for AR attributes, BulkDatabaseCapturer for SQL
+  # WHERE binds + SET values).
+  #
+  # This is a NAME-pattern denylist — the secondary defense, separate from
+  # the primary defense (Rails `encrypts :foo` introspection via
+  # `model.class.encrypted_attributes`, handled in EncryptedAttributes).
+  # Use both together: the encrypts check catches what the host app
+  # declared, this list catches what got past the declaration (legacy
+  # columns, manual hashing, externally-generated material).
+  #
+  # Matching rules:
+  # - Case-insensitive
+  # - Substring (so `customer_password` matches `password`)
+  # - User-extensible via `EzLogsAgent.configuration.excluded_graphql_variable_keys`
+  module SensitivePatterns
+    # Union of every column / key name we treat as sensitive. Curated
+    # from RFC 7468 / OWASP top sensitive-data categories plus
+    # ActiveRecord conventions. Keep this list narrow but defensive —
+    # adding a pattern is cheap; removing one is a backwards-incompatible
+    # behavior change for customer data on the wire.
+    PATTERNS = %w[
+      password passwd pwd
+      token access_token refresh_token api_token auth_token
+      secret api_secret client_secret
+      api_key apikey private_key privatekey secret_key secretkey
+      public_key signing_key
+      credential auth authorization
+      encrypted encrypted_data
+      pem cipher nonce salt digest signature hmac
+      ssn social_security
+      credit_card card_number cvv cvc
+    ].freeze
+
+    module_function
+
+    # @param key [String, Symbol, nil]
+    # @return [Boolean] true if the key matches a sensitive pattern OR
+    #   matches a user-configured pattern in `excluded_graphql_variable_keys`
+    def match?(key)
+      return false if key.nil?
+
+      key_lower = key.to_s.downcase
+      return true if PATTERNS.any? { |pattern| key_lower.include?(pattern) }
+
+      user_patterns = user_patterns_cache
+      user_patterns.any? { |pattern| key_lower.include?(pattern) }
+    rescue StandardError
+      # Defensive: if configuration access raises, treat as sensitive.
+      # Better to over-mask than to leak.
+      true
+    end
+
+    # Memoized lookup of user-configured patterns, already lowercased.
+    # Called per HTTP param, per DB attribute, per job arg key. Without
+    # the cache it would dispatch into EzLogsAgent.configuration on
+    # every check.
+    #
+    # The cache is keyed by `object_id` of the configured array, so
+    # `EzLogsAgent.configure { |c| c.excluded_graphql_variable_keys = [...] }`
+    # invalidates it naturally (assigning a new array changes its id).
+    # No explicit invalidation needed.
+    def user_patterns_cache
+      configured = EzLogsAgent.configuration.excluded_graphql_variable_keys
+      return @cached_user_patterns if configured.nil? && @cached_source_id.nil?
+
+      source_id = configured&.object_id
+      if source_id != @cached_source_id
+        @cached_user_patterns =
+          (configured || []).map { |p| p.to_s.downcase }.freeze
+        @cached_source_id = source_id
+      end
+      @cached_user_patterns
+    end
+    private_class_method :user_patterns_cache
+  end
+end

data/lib/ez_logs_agent/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module EzLogsAgent
-  VERSION = "0.1.10"
+  VERSION = "0.2.1"
 end

data/lib/ez_logs_agent.rb

@@ -8,6 +8,9 @@ require_relative "ez_logs_agent/correlation"
 require_relative "ez_logs_agent/actor_validator"
 require_relative "ez_logs_agent/actor"
 require_relative "ez_logs_agent/user_agent_detector"
+require_relative "ez_logs_agent/sensitive_patterns"
+require_relative "ez_logs_agent/encrypted_attributes"
+require_relative "ez_logs_agent/bulk_sql_parser"
 require_relative "ez_logs_agent/sanitizer"
 require_relative "ez_logs_agent/event_builder"
 require_relative "ez_logs_agent/resource_extractor"
@@ -19,6 +22,7 @@ require_relative "ez_logs_agent/middleware/http_request"
 require_relative "ez_logs_agent/capturers/job_capturer"
 require_relative "ez_logs_agent/capturers/active_job_capturer"
 require_relative "ez_logs_agent/capturers/database_capturer"
+require_relative "ez_logs_agent/capturers/bulk_database_capturer"
 
 # Load Railtie only when Rails is present
 require_relative "ez_logs_agent/railtie" if defined?(Rails::Railtie)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ez_logs_agent
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.2.1
 platform: ruby
 authors:
 - dezsirazvan
@@ -123,12 +123,15 @@ files:
 - lib/ez_logs_agent/actor.rb
 - lib/ez_logs_agent/actor_validator.rb
 - lib/ez_logs_agent/buffer.rb
+- lib/ez_logs_agent/bulk_sql_parser.rb
 - lib/ez_logs_agent/capturers/active_job_capturer.rb
+- lib/ez_logs_agent/capturers/bulk_database_capturer.rb
 - lib/ez_logs_agent/capturers/database_capturer.rb
 - lib/ez_logs_agent/capturers/job_capturer.rb
 - lib/ez_logs_agent/configuration.rb
 - lib/ez_logs_agent/configuration_validator.rb
 - lib/ez_logs_agent/correlation.rb
+- lib/ez_logs_agent/encrypted_attributes.rb
 - lib/ez_logs_agent/event_builder.rb
 - lib/ez_logs_agent/flush_scheduler.rb
 - lib/ez_logs_agent/logger.rb
@@ -137,6 +140,7 @@ files:
 - lib/ez_logs_agent/resource_extractor.rb
 - lib/ez_logs_agent/retry_sender.rb
 - lib/ez_logs_agent/sanitizer.rb
+- lib/ez_logs_agent/sensitive_patterns.rb
 - lib/ez_logs_agent/transport.rb
 - lib/ez_logs_agent/user_agent_detector.rb
 - lib/ez_logs_agent/version.rb