cache_sweeper 0.1.0

data/lib/cache_sweeper/async_worker.rb

@@ -0,0 +1,121 @@
+# Sidekiq worker for async cache invalidation
+module CacheSweeper
+  class AsyncWorker
+    if defined?(Sidekiq)
+      include Sidekiq::Worker
+    end
+
+    def perform(keys, trigger = :instant)
+      start_time = Time.current
+      keys_array = Array(keys)
+      log_job_start(keys_array, trigger)
+
+      deleted_count = CacheSweeper.delete_cache_keys(keys_array, {
+        job_id: jid,
+        mode: :async,
+        trigger: trigger
+      })
+      failed_count = keys_array.length - deleted_count
+
+      duration = (Time.current - start_time) * 1000
+      log_job_completion(keys_array, deleted_count, failed_count, duration, [], trigger)
+
+    rescue => e
+      duration = (Time.current - start_time) * 1000
+      log_job_error(e, keys_array, duration, trigger)
+      raise e
+    end
+
+    def self.perform_async(keys, trigger = :instant)
+      if defined?(Sidekiq) && self.ancestors.include?(Sidekiq::Worker)
+        log_job_scheduling(keys, :sidekiq, trigger)
+        self.set(sidekiq_opts).super(keys, trigger)
+      else
+        # In test environment or when Sidekiq is not available, perform synchronously
+        log_job_scheduling(keys, :synchronous, trigger)
+        new.perform(keys, trigger)
+      end
+    end
+
+    private
+
+    def log_job_start(keys, trigger)
+      CacheSweeper::Logger.log_async_jobs("Async job started", :info, {
+        job_id: jid,
+        keys_count: keys.length,
+        keys: keys,
+        trigger: trigger
+      })
+    end
+
+    def log_cache_deletion(keys, status, error = nil)
+      case status
+      when :success
+        CacheSweeper::Logger.log_async_jobs("Cache deleted: #{Array(keys).length} keys", :debug, {
+          job_id: jid,
+          keys_count: Array(keys).length,
+          keys: Array(keys),
+          status: 'success'
+        })
+      when :rails_not_available
+        CacheSweeper::Logger.log_async_jobs("Rails cache not available for #{Array(keys).length} keys", :warn, {
+          job_id: jid,
+          keys_count: Array(keys).length,
+          keys: Array(keys),
+          status: 'rails_not_available'
+        })
+      when :error
+        CacheSweeper::Logger.log_error(error, {
+          job_id: jid,
+          keys_count: Array(keys).length,
+          keys: Array(keys),
+          status: 'error',
+          error_type: 'cache_delete_error'
+        })
+      end
+    end
+
+    def log_job_completion(keys, deleted_count, failed_count, duration, errors, trigger)
+      CacheSweeper::Logger.log_async_jobs("Async job completed", :info, {
+        job_id: jid,
+        keys_count: keys.length,
+        deleted_count: deleted_count,
+        failed_count: failed_count,
+        duration_ms: duration.round(3),
+        errors: errors,
+        trigger: trigger
+      })
+
+      CacheSweeper::Logger.log_performance("async_cache_deletion", duration, {
+        job_id: jid,
+        keys_count: keys.length,
+        deleted_count: deleted_count,
+        failed_count: failed_count,
+        trigger: trigger
+      })
+    end
+
+    def log_job_error(error, keys, duration, trigger)
+      CacheSweeper::Logger.log_error(error, {
+        job_id: jid,
+        keys: keys,
+        duration_ms: duration.round(3),
+        error_type: 'async_job_error',
+        trigger: trigger
+      })
+    end
+
+    def self.log_job_scheduling(keys, method, trigger)
+      CacheSweeper::Logger.log_async_jobs("Async job scheduled", :info, {
+        method: method,
+        keys_count: Array(keys).length,
+        keys: Array(keys),
+        trigger: trigger
+      })
+    end
+
+    def jid
+      @jid ||= SecureRandom.hex(8)
+    end
+  end
+end

data/lib/cache_sweeper/base.rb

@@ -0,0 +1,34 @@
+# Base class for all sweepers
+module CacheSweeper
+  class Base
+    include CacheSweeper::DSL
+
+    class << self
+      # Sweeper-level configuration
+      attr_accessor :trigger, :mode, :queue, :sidekiq_options
+
+      # Clean DSL for sweeper configuration
+      def sweeper_options(options = {})
+        @trigger = options[:trigger] if options.key?(:trigger)
+        @mode = options[:mode] if options.key?(:mode)
+        @queue = options[:queue] if options.key?(:queue)
+        @sidekiq_options = options[:sidekiq_options] if options.key?(:sidekiq_options)
+        CacheSweeper.validate_async_mode(@mode, "sweeper #{self.name}")
+      end
+
+    end
+
+
+    # Instance method support for custom conditions
+    def call_condition(condition, *args)
+      case condition
+      when Proc
+        instance_exec(*args, &condition)
+      when Symbol, String
+        send(condition, *args)
+      else
+        true
+      end
+    end
+  end
+end

data/lib/cache_sweeper/dsl.rb

@@ -0,0 +1,32 @@
+# lib/cache_sweeper/dsl.rb
+module CacheSweeper
+  module DSL
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    module ClassMethods
+      def watch(association = nil, attributes: nil, if: nil, keys: nil, trigger: nil, mode: nil, queue: nil, sidekiq_options: nil, callback: nil, on: nil)
+        CacheSweeper.validate_async_mode(mode, "rule in #{self.name}")
+        @cache_sweeper_rules ||= []
+        @cache_sweeper_rules << {
+          association: association,
+          attributes: attributes,
+          condition: binding.local_variable_get(:if),
+          keys: keys,
+          trigger: trigger,
+          mode: mode,
+          queue: queue,
+          sidekiq_options: sidekiq_options,
+          callback: callback,
+          on: on,
+          sweeper_class: self
+        }
+      end
+
+      def cache_sweeper_rules
+        @cache_sweeper_rules || []
+      end
+    end
+  end
+end

data/lib/cache_sweeper/flush_middleware.rb

@@ -0,0 +1,152 @@
+require 'request_store'
+
+# Middleware to flush pending cache keys at the end of each request if batching is enabled
+class CacheSweeperFlushMiddleware
+  def initialize(app)
+    @app = app
+  end
+
+  def call(env)
+    start_time = Time.current
+    request_id = SecureRandom.hex(8)
+
+    log_request_start(request_id, env)
+
+    status, headers, response = @app.call(env)
+
+    # At end of request, flush all request-level batched keys
+    pending = RequestStore.store[:cache_sweeper_request_pending] || []
+
+    if pending.any?
+      log_request_flush_start(request_id, pending)
+
+      async_jobs_scheduled = 0
+      instant_deletions = 0
+      failed_deletions = 0
+      errors = []
+
+      pending.each do |entry|
+        keys, mode, sidekiq_options = entry.values_at(:keys, :mode, :sidekiq_options)
+
+        begin
+          if mode == :async
+            CacheSweeper::AsyncWorker.set(sidekiq_options || {}).perform_async(keys, :request)
+            async_jobs_scheduled += 1
+            log_batch_processing(request_id, keys, :async_scheduled, sidekiq_options)
+          else
+            deleted_count = CacheSweeper.delete_cache_keys(keys, {
+              request_id: request_id,
+              mode: :inline,
+              trigger: :request
+            })
+            instant_deletions += deleted_count
+            failed_deletions += Array(keys).length - deleted_count
+          end
+        rescue => e
+          errors << { keys: keys, error: e.message }
+          log_batch_processing(request_id, keys, :batch_error, sidekiq_options, e)
+        end
+      end
+
+      log_request_flush_completion(request_id, {
+        total_batches: pending.length,
+        async_jobs_scheduled: async_jobs_scheduled,
+        instant_deletions: instant_deletions,
+        failed_deletions: failed_deletions,
+        errors: errors
+      })
+    else
+      log_request_no_flush(request_id)
+    end
+
+    [status, headers, response]
+  rescue => e
+    duration = (Time.current - start_time) * 1000
+    log_request_error(request_id, e, duration)
+    raise e
+  ensure
+    RequestStore.store[:cache_sweeper_request_pending] = []
+    duration = (Time.current - start_time) * 1000
+    log_request_completion(request_id, duration)
+  end
+
+  private
+
+  def log_request_start(request_id, env)
+    CacheSweeper::Logger.log_middleware("Request started", :debug, {
+      request_id: request_id,
+      method: env['REQUEST_METHOD'],
+      path: env['PATH_INFO']
+    })
+  end
+
+  def log_request_flush_start(request_id, pending)
+    total_keys = pending.sum { |entry| Array(entry[:keys]).length }
+    async_count = pending.count { |entry| entry[:async] }
+    instant_count = pending.length - async_count
+
+    CacheSweeper::Logger.log_middleware("Request flush started", :info, {
+      request_id: request_id,
+      batch_count: pending.length,
+      total_keys: total_keys,
+      async_batches: async_count,
+      instant_batches: instant_count
+    })
+  end
+
+  def log_batch_processing(request_id, keys, status, sidekiq_options = nil, error = nil)
+    case status
+    when :async_scheduled
+      CacheSweeper::Logger.log_middleware("Batch scheduled async", :debug, {
+        request_id: request_id,
+        keys: Array(keys),
+        sidekiq_options: sidekiq_options
+      })
+    when :rails_not_available
+      CacheSweeper::Logger.log_middleware("Rails cache not available", :warn, {
+        request_id: request_id,
+        keys: Array(keys)
+      })
+    when :error
+      CacheSweeper::Logger.log_error(error, {
+        request_id: request_id,
+        keys: Array(keys),
+        error_type: 'batch_processing_error'
+      })
+    when :batch_error
+      CacheSweeper::Logger.log_error(error, {
+        request_id: request_id,
+        keys: Array(keys),
+        sidekiq_options: sidekiq_options,
+        error_type: 'batch_error'
+      })
+    end
+  end
+
+  def log_request_flush_completion(request_id, stats)
+    CacheSweeper::Logger.log_middleware("Request flush completed", :info, {
+      request_id: request_id,
+      **stats
+    })
+  end
+
+  def log_request_no_flush(request_id)
+    CacheSweeper::Logger.log_middleware("No cache flush needed", :debug, {
+      request_id: request_id
+    })
+  end
+
+  def log_request_error(request_id, error, duration)
+    CacheSweeper::Logger.log_error(error, {
+      request_id: request_id,
+      duration_ms: duration.round(3),
+      error_type: 'middleware_error'
+    })
+  end
+
+  def log_request_completion(request_id, duration)
+    CacheSweeper::Logger.log_performance("request_processing", duration, {
+      request_id: request_id
+    })
+  end
+end

data/lib/cache_sweeper/loader.rb

@@ -0,0 +1,368 @@
+require 'request_store'
+require 'set'
+
+module CacheSweeper
+  module Loader
+    SWEEPER_PATH = 'app/sweepers'.freeze
+
+    class << self
+      # Configuration resolution methods
+      def resolve_trigger(rule, sweeper)
+        rule[:trigger] || sweeper.trigger || CacheSweeper.trigger
+      end
+
+      def resolve_mode(rule, sweeper)
+        rule[:mode] || sweeper.mode || CacheSweeper.mode
+      end
+
+      def resolve_queue(rule, sweeper)
+        rule[:queue] || sweeper.queue || CacheSweeper.queue
+      end
+
+      def resolve_sidekiq_options(rule, sweeper)
+        # Merge options in order of precedence: rule > sweeper > global
+        options = CacheSweeper.sidekiq_options.dup
+        options.merge!(sweeper.sidekiq_options) if sweeper.sidekiq_options
+        options.merge!(rule[:sidekiq_options]) if rule[:sidekiq_options]
+
+        # Add queue to options if specified
+        queue = resolve_queue(rule, sweeper)
+        options[:queue] = queue if queue && queue != :default
+
+        options
+      end
+    end
+
+    def self.collect_pending_key(key, sweeper = nil, mode = :request)
+      if mode == :request
+        RequestStore.store["cache_sweeper_pending_keys_#{sweeper&.name || 'global'}"] ||= Set.new
+        RequestStore.store["cache_sweeper_pending_keys_#{sweeper&.name || 'global'}"] << key
+      elsif mode == :job
+        RequestStore.store["cache_sweeper_job_keys_#{sweeper&.name || 'global'}"] ||= Set.new
+        RequestStore.store["cache_sweeper_job_keys_#{sweeper&.name || 'global'}"] << key
+      end
+    end
+
+    def self.flush_pending_keys(sweeper: nil)
+      start_time = Time.current
+      sweeper_name = sweeper&.name || 'global'
+
+      CacheSweeper::Logger.log_cache_operations("Flushing pending keys for sweeper: #{sweeper_name}", :debug, {
+        sweeper: sweeper_name
+      })
+
+      # Flush request-level keys
+      request_key = "cache_sweeper_pending_keys_#{sweeper_name}"
+      keys = RequestStore.store[request_key]
+      if keys&.any?
+        if defined?(Rails) && Rails.respond_to?(:cache)
+          deleted_count = CacheSweeper.delete_cache_keys(keys, {
+            sweeper: sweeper_name,
+            mode: :inline,
+            trigger: :request
+          })
+          failed_count = keys.length - deleted_count
+        else
+          deleted_count = 0
+          failed_count = keys.length
+        end
+
+        CacheSweeper::Logger.log_cache_operations("Flushed request-level keys", :info, {
+          sweeper: sweeper_name,
+          deleted_count: deleted_count,
+          failed_count: failed_count,
+          total_keys: keys.length
+        })
+
+        RequestStore.store[request_key] = Set.new
+      end
+
+      # Flush job-level keys
+      job_key = "cache_sweeper_job_keys_#{sweeper_name}"
+      job_keys = RequestStore.store[job_key]
+      if job_keys&.any?
+        begin
+          CacheSweeper::AsyncWorker.perform_async(job_keys.to_a, :request)
+          CacheSweeper::Logger.log_cache_operations("Scheduled async job for pending keys", :info, {
+            sweeper: sweeper_name,
+            keys_count: job_keys.length,
+            keys: job_keys.to_a
+          })
+        rescue => e
+          CacheSweeper::Logger.log_error(e, {
+            sweeper: sweeper_name,
+            keys: job_keys.to_a,
+            error_type: 'async_job_schedule_error'
+          })
+        end
+        RequestStore.store[job_key] = Set.new
+      end
+
+      duration = (Time.current - start_time) * 1000
+      CacheSweeper::Logger.log_performance("flush_pending_keys", duration, {
+        sweeper: sweeper_name,
+        request_keys_flushed: keys&.length || 0,
+        job_keys_scheduled: job_keys&.length || 0
+      })
+    end
+
+    def self.load_sweepers!
+      Dir[Rails.root.join(SWEEPER_PATH, '**', '*_sweeper.rb')].each { |file| require_dependency file }
+    end
+
+    def self.hook_sweepers!
+      start_time = Time.current
+      CacheSweeper::Logger.log_initialization("Starting model attachment process")
+
+      sweeper_count = 0
+      rule_count = 0
+      error_count = 0
+      CacheSweeper::Base.descendants.each do |sweeper|
+        sweeper_count += 1
+        CacheSweeper::Logger.log_initialization("Processing sweeper: #{sweeper.name}", { sweeper: sweeper.name, rule_count: sweeper.cache_sweeper_rules.length })
+
+        sweeper.cache_sweeper_rules.each do |rule|
+          rule_count += 1
+          association = rule[:association]
+          attributes = rule[:attributes]
+          condition = rule[:condition]
+          keys = rule[:keys]
+
+          begin
+            if association.nil?
+              model_name = sweeper.name.sub('Sweeper', '')
+              model = model_name.constantize
+              CacheSweeper::Logger.log_initialization("Attaching direct model callback: #{model_name}", {
+                sweeper: sweeper.name,
+                model: model_name,
+                attributes: attributes,
+                callback: rule[:callback] || :after_commit,
+                events: rule[:on] || [:create, :update, :destroy]
+              })
+              attach_callbacks(model, sweeper, attributes, condition, keys, rule)
+            else
+              parent_model_name = sweeper.name.sub('Sweeper', '')
+              parent_model = parent_model_name.constantize
+              assoc_reflection = parent_model.reflect_on_association(association)
+
+              unless assoc_reflection
+                CacheSweeper::Logger.warn("Association not found: #{parent_model_name}##{association}", {
+                  sweeper: sweeper.name,
+                  parent_model: parent_model_name,
+                  association: association
+                })
+                next
+              end
+
+              assoc_model = assoc_reflection.klass
+              CacheSweeper::Logger.log_initialization("Attaching association callback: #{parent_model_name}##{association} -> #{assoc_model.name}", {
+                sweeper: sweeper.name,
+                parent_model: parent_model_name,
+                association: association,
+                assoc_model: assoc_model.name,
+                attributes: attributes,
+                callback: rule[:callback] || :after_commit,
+                events: rule[:on] || [:create, :update, :destroy]
+              })
+              attach_callbacks(assoc_model, sweeper, attributes, condition, keys, rule, parent_model, association)
+            end
+          rescue NameError => e
+            error_count += 1
+            CacheSweeper::Logger.log_error(e, {
+              sweeper: sweeper.name,
+              association: association,
+              error_type: 'model_not_found'
+            })
+            # Model doesn't exist yet, skip for now
+            next
+          rescue => e
+            error_count += 1
+            CacheSweeper::Logger.log_error(e, {
+              sweeper: sweeper.name,
+              association: association,
+              error_type: 'attachment_error'
+            })
+            next
+          end
+        end
+      end
+
+      duration = (Time.current - start_time) * 1000
+      CacheSweeper::Logger.log_initialization("Model attachment completed", {
+        sweeper_count: sweeper_count,
+        rule_count: rule_count,
+        error_count: error_count,
+        duration_ms: duration.round(3)
+      })
+    end
+
+    def self.attach_callbacks(model, sweeper, attributes, condition, keys, rule, parent_model = nil, association = nil)
+      callback = lambda do |record|
+        start_time = Time.current
+        sweeper_instance = sweeper.new
+
+        # Log rule execution start
+        CacheSweeper::Logger.log_rule_execution(rule, record, 'started', {
+          event: record.previous_changes.keys.any? ? 'update' : (record.persisted? ? 'create' : 'destroy'),
+          changed_attributes: record.saved_changes.keys
+        })
+
+        begin
+          # Check attribute changes
+          if attributes
+            changed = (record.saved_changes.keys.map(&:to_sym) & attributes.map(&:to_sym)).any?
+            CacheSweeper::Logger.log_rule_execution(rule, record, "attribute_check: #{changed}", {
+              watched_attributes: attributes,
+              changed_attributes: record.saved_changes.keys,
+              relevant_changes: record.saved_changes.keys.map(&:to_sym) & attributes.map(&:to_sym)
+            })
+            next unless changed
+          end
+
+          # Check condition
+          if condition
+            condition_result = sweeper_instance.call_condition(condition, record)
+            CacheSweeper::Logger.log_rule_execution(rule, record, "condition_check: #{condition_result}", {
+              condition: condition.class.name,
+              condition_result: condition_result
+            })
+            next unless condition_result
+          end
+
+          # Generate cache keys
+          cache_keys = if keys.is_a?(Proc)
+            begin
+              generated_keys = keys.call(record)
+              CacheSweeper::Logger.log_rule_execution(rule, record, "keys_generated", {
+                keys_count: Array(generated_keys).length,
+                keys: Array(generated_keys)
+              })
+              generated_keys
+            rescue => e
+              CacheSweeper::Logger.log_error(e, {
+                sweeper: sweeper.name,
+                record_class: record.class.name,
+                record_id: record.id,
+                error_type: 'key_generation_error'
+              })
+              next
+            end
+          else
+            Array(keys)
+          end
+
+          # Execute cache invalidation
+          if parent_model && association
+            # For association rules, we need to find the parent records
+            # that have this record in their association
+            parents = parent_model.joins(association).where(association => { id: record.id })
+            CacheSweeper::Logger.log_rule_execution(rule, record, "association_processing", {
+              parent_count: parents.count,
+              association: association
+            })
+            Array(parents).each do |parent|
+              CacheSweeper::Loader.invalidate_cache(cache_keys, record, rule)
+            end
+          else
+            CacheSweeper::Loader.invalidate_cache(cache_keys, record, rule)
+          end
+
+          # Log successful completion
+          duration = (Time.current - start_time) * 1000
+          CacheSweeper::Logger.log_rule_execution(rule, record, "completed", {
+            duration_ms: duration.round(3),
+            cache_keys_processed: Array(cache_keys).length
+          })
+
+        rescue => e
+          duration = (Time.current - start_time) * 1000
+          CacheSweeper::Logger.log_error(e, {
+            sweeper: sweeper.name,
+            record_class: record.class.name,
+            record_id: record.id,
+            duration_ms: duration.round(3),
+            error_type: 'rule_execution_error'
+          })
+        end
+      end
+
+      callback_type = rule[:callback] || :after_commit
+      events = rule[:on] || [:create, :update, :destroy]
+      model.send(callback_type, callback, on: events)
+    end
+
+    def self.invalidate_cache(keys, *args, rule)
+      start_time = Time.current
+      sweeper = rule[:sweeper_class]
+      trigger = resolve_trigger(rule, sweeper)
+      mode = resolve_mode(rule, sweeper)
+      sidekiq_opts = resolve_sidekiq_options(rule, sweeper)
+
+      CacheSweeper::Logger.debug("Cache invalidation started", {
+        sweeper: sweeper&.name,
+        trigger: trigger,
+        mode: mode,
+        keys_count: Array(keys).length
+      })
+
+      begin
+        if trigger == :request
+          RequestStore.store[:cache_sweeper_request_pending] ||= []
+          RequestStore.store[:cache_sweeper_request_pending] << { keys: keys, mode: mode, sidekiq_options: sidekiq_opts }
+          CacheSweeper::Logger.log_cache_operations("Batched for request: #{Array(keys).inspect} (mode: #{mode})", :info, {
+            keys: Array(keys),
+            mode: mode,
+            batch_size: RequestStore.store[:cache_sweeper_request_pending].length
+          })
+        else
+          if mode == :async
+            CacheSweeper::AsyncWorker.set(sidekiq_opts).perform_async(keys, trigger)
+            CacheSweeper::Logger.log_cache_operations("Scheduled async job for: #{Array(keys).inspect}", :info, {
+              keys: Array(keys),
+              sidekiq_options: sidekiq_opts
+            })
+          else
+            if defined?(Rails) && Rails.respond_to?(:cache)
+              deleted_count = CacheSweeper.delete_cache_keys(keys, {
+                sweeper: sweeper&.name,
+                mode: :inline,
+                trigger: :instant
+              })
+              failed_count = Array(keys).length - deleted_count
+            else
+              CacheSweeper::Logger.log_cache_operations("Rails cache not available for #{Array(keys).length} keys", :warn, {
+                keys: Array(keys)
+              })
+              deleted_count = 0
+              failed_count = Array(keys).length
+            end
+
+            CacheSweeper::Logger.log_cache_operations("Instant deletion completed", :info, {
+              deleted_count: deleted_count,
+              failed_count: failed_count,
+              total_keys: Array(keys).length
+            })
+          end
+        end
+
+        duration = (Time.current - start_time) * 1000
+        CacheSweeper::Logger.log_performance("cache_invalidation", duration, {
+          trigger: trigger,
+          mode: mode,
+          keys_count: Array(keys).length
+        })
+
+      rescue => e
+        duration = (Time.current - start_time) * 1000
+        CacheSweeper::Logger.log_error(e, {
+          sweeper: sweeper&.name,
+          trigger: trigger,
+          mode: mode,
+          keys: Array(keys),
+          duration_ms: duration.round(3),
+          error_type: 'cache_invalidation_error'
+        })
+      end
+    end
+  end
+end