has_helpers 5.0.2

data/app/workers/has_helpers/db_operations/clean_stats_worker.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  class DBOperations::CleanStatsWorker < HasUtils::GoodJob::BaseWorker
+    include HasHelpers::DefaultWorker
+    include HasHelpers::RedisConfigurable
+
+    queue_as :db_clean
+
+    AGE_LIMIT_STRING = "1 month"
+
+    config_defaults(
+      db_operation_stats_resume_enabled: true # means db_operations_stats_resumes was created and is used to reporting purposes
+    )
+
+    def perform
+      ::Rails.logger.info "Removing db_operations_stats older than #{AGE_LIMIT_STRING}...."
+
+      total_removed = 0
+
+      loop do
+        num_removed = ::HasHelpers::DBOperations::Operations.with_advisory_lock("'db_operations_stats'::regclass::integer") do |connection|
+          connection.exec_delete <<-END_OF_SQL
+            DELETE FROM db_operations_stats
+            WHERE created_at <= now() - interval '#{AGE_LIMIT_STRING}'
+          END_OF_SQL
+        end
+
+        break if num_removed == 0
+        total_removed += num_removed
+      end
+
+      ::Rails.logger.info "Removed #{total_removed} operation stats."
+
+      if config.db_operation_stats_resume_enabled
+        HasHelpers::DBOperations::UpdateOperationsStatsResumeWorker.perform_later
+      end
+    end
+
+    delegate :config, to: :class
+  end
+end

data/app/workers/has_helpers/db_operations/helper_worker.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  class DBOperations::HelperWorker < HasUtils::GoodJob::BaseWorker
+    include HasHelpers::DefaultWorker
+
+    queue_as :pending_db_ops
+
+    MAIN_KLASS = DBOperations::MainWorker
+
+    # queue_type is mandatory and could be one of the following:
+    # "fast"         runs the worker for the fast queries queue
+    # "slow"         runs the worker for the slow queries queue
+    def perform(queue_type)
+      return if HasHelpers::Feature.active?(:old_pending_db_ops)
+
+      if ::HasHelpers::DBOperations::Operations.paused?
+        ::Rails.logger.info "DBOperations::HelperWorker skip because paused"
+        return
+      end
+
+      # Use appropriate lock and queue names depending on the queue type
+      lock_key, queue_name, is_fast = determine_queue(queue_type)
+
+      # Check if another worker is running for this queue; if so, exit early
+      return if redis.get(lock_key) == "true"
+
+      # Set a lock in Redis to indicate that this worker is running, with an expiration time
+      redis.set(lock_key, "true", ex: MAIN_KLASS.config.helper_worker_lock_expiration)
+
+      # Start renewing the lock in a separate thread (for long-running queries)
+      lock_renewal_thread = Thread.new { renew_lock_periodically(lock_key) }
+
+      begin
+        # Retrieve and remove a batch of jobs from Redis in an atomic operation
+        ids = redis.zpopmin(queue_name, MAIN_KLASS.config.helper_worker_batch_size).map { |id, _score| id }
+
+        return if ids.empty? # If no jobs are left in Redis, exit
+
+        triggered_slow_query_worker = false
+
+        ids.each do |id|
+          job = HasHelpers::PendingDBOperation.find_by(id: id)
+          begin
+            # Process the query (with timeout for fast queries)
+            execute_query(job, is_fast) if job
+          rescue ::PsqlHq::StatementTimeout
+            if is_fast
+              # If a query timeout occurs, move it to the slow queue
+              mark_as_slow(job)
+
+              # Trigger the slow worker if needed and only once for the whole batch
+              if redis.zcard(MAIN_KLASS::SLOW_QUEUE) > 0 && !triggered_slow_query_worker
+                ::HasHelpers::DBOperations::HelperWorker.perform_later("slow")
+                triggered_slow_query_worker = true
+              end
+            end
+          end
+        end
+
+        # Re-enqueue this worker for fast or slow queue if more jobs remain
+        ::HasHelpers::DBOperations::HelperWorker.perform_later(queue_type) if redis.zcard(queue_name) > 0
+      ensure
+        # Stop lock renewal and clear the lock in Redis when the worker finishes
+        lock_renewal_thread.kill
+        redis.del(lock_key)
+      end
+    end
+
+    # Helper method to determine queue type, name and lock key
+    def determine_queue(queue_type)
+      if queue_type == "fast"
+        [MAIN_KLASS::FAST_WORKER_LOCK_KEY, MAIN_KLASS::FAST_QUEUE, true]
+      else
+        [MAIN_KLASS::SLOW_WORKER_LOCK_KEY, MAIN_KLASS::SLOW_QUEUE, false]
+      end
+    end
+
+    # Helper method to execute the query, we establish a timeout for fast queues
+    # for slow queues we just execute the query without timeout, since we don't know how much they might take
+    # The idea is that slow queries should be a rare case and not a common scenario,  so we let them be
+    # until someone fix them
+    def execute_query(job, is_fast)
+      ::PsqlHq::TimeoutRunner.within(timeout: is_fast ? MAIN_KLASS.config.helper_worker_query_timeout_fast : MAIN_KLASS.config.helper_worker_query_timeout_slow) do |connection|
+        job.execute(connection: connection)
+      end
+    end
+
+    # Mark the query as slow
+    def mark_as_slow(job)
+      remove_job_from_queue(MAIN_KLASS::FAST_QUEUE, job.id)
+      job.defer_processing # defer processing so next time is going to be enqueued in the slow queue
+      job.is_slow = true
+      job.save
+      log_slow_query(job)
+    end
+
+    # Log slow query details for later analysis
+    def log_slow_query(job)
+      time = Time.zone.now
+
+      # TODO fix this when this is implemented in agencies
+      # ex: agencies has its own stat class and field names are different
+      stat = ::HasHelpers::Stat.new
+      stat.name = job.relation_name
+      stat.maintype = "pending_db_ops"
+      stat.subtype = "slow"
+      stat.data = { id: job.id, query: job.sql, foreign_key: job.foreign_key_value }
+      stat.created_at = time
+      stat.updated_at = time
+      stat.save!
+    end
+
+    # Remove a job from Redis queue
+    def remove_job_from_queue(queue_name, job_id)
+      redis.zrem(queue_name, job_id)
+    end
+
+    # Renew the lock periodically (every 4 minutes)
+    def renew_lock_periodically(lock_key)
+      loop do
+        sleep 1.minute
+        redis.expire(lock_key, MAIN_KLASS.config.helper_worker_lock_expiration) if redis.get(lock_key)
+      end
+    end
+
+    def redis
+      RedisConf.current
+    end
+  end
+end

data/app/workers/has_helpers/db_operations/main_worker.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  class DBOperations::MainWorker < HasUtils::GoodJob::BaseWorker
+    include HasHelpers::DefaultWorker
+    include HasHelpers::RedisConfigurable
+
+    queue_as :pending_db_ops
+
+    FAST_QUEUE = "pending_db_ops_fast" # The queue used for fast/usual jobs
+    SLOW_QUEUE = "pending_db_ops_slow" # The queue used for jobs that were marked as slow
+    FAST_WORKER_LOCK_KEY = "pending_db_ops_fast_worker" # The lock key for the worker that executes jobs in the fast queue
+    SLOW_WORKER_LOCK_KEY = "pending_db_ops_slow_worker" # The lock key for the worker that executes jobs in the slow queue
+    MAIN_WORKER_LOCK_KEY = "pending_db_ops_main_worker" # The lock key for the main worker
+
+    config_defaults(
+      enqueue_batch_size: 1000, # How many jobs the main worker enqueues each time
+      max_queue_size: 3000, # The max number of jobs the queue will hold
+      max_enqueue_time: 10.minutes, # How long a job is going to be in the queue before considering it stucked and re-enqueue it
+      helper_worker_lock_expiration: 5.minutes, # How long each worker holds the key (to avoid running multiple workers of the same kind at the same time)
+      helper_worker_batch_size: 1000, # Batch size for helper workers
+      helper_worker_query_timeout_fast: 1.second, # Max time we'll allow normal queries to execute
+      helper_worker_query_timeout_slow: 5.minutes, # Max time we'll allow slow queries to execute
+      # If there are no more jobs to be enqueued or if the queue is full, the main worker is going to slee
+      # for `main_worker_sleep` long before trying to search for more pending db ops, as long as no more than
+      # `main_worker_lifespan` time has elapsed,  in which case the worker is going to be finished until it
+      # is re-enqueued by the sidekiq worker in the next minute.
+      # The reason behind doing this is that sometimes the main worker can't enqueue more jobs and if it finishes,
+      # it won't be executed until the next minute (by the sidekiq scheduler). By doing this we try to reduce the
+      #  number of pending db ops that get stack before being enqueued for processing
+      main_worker_sleep: 10.seconds,
+      main_worker_lifespan: 3.minutes
+    )
+
+    def perform
+      return if HasHelpers::Feature.active?(:old_pending_db_ops)
+
+      if ::HasHelpers::DBOperations::Operations.paused?
+        ::Rails.logger.info "DBOperations::MainWorker skip because paused"
+        return
+      end
+
+      begin
+        return if main_worker_locked?
+        started_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+        loop do
+          lock_main_worker
+          main_process
+
+          ended_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+          sleep(config.main_worker_sleep.seconds)
+          break if (ended_at - started_at) > config.main_worker_lifespan.seconds
+          break if ::HasHelpers::DBOperations::Operations.paused?
+        end
+      ensure
+        release_main_worker_lock
+      end
+    end
+
+    def main_worker_locked?
+      redis.get(MAIN_WORKER_LOCK_KEY) == "true"
+    end
+
+    def lock_main_worker
+      if redis.get(MAIN_WORKER_LOCK_KEY) == "true"
+        redis.expire(MAIN_WORKER_LOCK_KEY, 1.minute)
+      else
+        redis.set(MAIN_WORKER_LOCK_KEY, "true", ex: 1.minute)
+      end
+    end
+
+    def release_main_worker_lock
+      redis.del(MAIN_WORKER_LOCK_KEY)
+    end
+
+    def main_process
+      # Recover jobs stuck in both fast and slow queues
+      recover_all_stuck_jobs
+
+      # Enqueue pending jobs in both fast and slow queues
+      enqueue_all_pending_jobs
+
+      # If there are jobs in the fast queue and no fast worker is running, trigger the fast worker
+      if redis.zcard(FAST_QUEUE) > 0 && !worker_running?(FAST_WORKER_LOCK_KEY)
+        ::HasHelpers::DBOperations::HelperWorker.perform_later("fast")
+      end
+
+      # If there are jobs in the slow queue and no slow worker is running, trigger the slow worker
+      if redis.zcard(SLOW_QUEUE) > 0 && !worker_running?(SLOW_WORKER_LOCK_KEY)
+        ::HasHelpers::DBOperations::HelperWorker.perform_later("slow")
+      end
+    end
+
+    def recover_all_stuck_jobs
+      recover_stuck_jobs(nil) # recover fast stuck queries
+      recover_stuck_jobs(true) # recover slow stuck queries
+    end
+
+    # Recover stuck jobs in the queue and re-enqueue them
+    def recover_stuck_jobs(is_slow)
+      queue_name = is_slow ? SLOW_QUEUE : FAST_QUEUE
+      current_queue_size = redis.zcard(queue_name)
+      return if current_queue_size >= config.max_queue_size
+
+      current_limit = [config.enqueue_batch_size, config.max_queue_size - current_queue_size].min
+
+      stuck_jobs_sql = HasHelpers::PendingDBOperation.where(HasHelpers::PendingDBOperation.arel_table[:enqueued_at].lt(Time.zone.now - config.max_enqueue_time.seconds)).where.not(enqueued_at: nil).where(is_slow: is_slow).limit(current_limit).to_sql
+      HasHelpers::PendingDBOperation.uncached do
+        stuck_jobs = HasHelpers::PendingDBOperation.find_by_sql(stuck_jobs_sql) # The reason behind doing this is to bypass ActiveRecord cache, I tried with just `uncached` but Rails was still caching the result
+        enqueue_jobs(stuck_jobs)
+      end
+    end
+
+    def enqueue_all_pending_jobs
+      enqueue_pending_jobs(nil) # enqueue fast queries
+      enqueue_pending_jobs(true) # enqueue slow queries
+    end
+
+    # Enqueue new jobs that have not been enqueued yet
+    def enqueue_pending_jobs(is_slow)
+      queue_name = is_slow ? SLOW_QUEUE : FAST_QUEUE
+      current_queue_size = redis.zcard(queue_name)
+      return if current_queue_size >= config.max_queue_size
+
+      current_limit = [config.enqueue_batch_size, config.max_queue_size - current_queue_size].min
+
+      pending_jobs_sql = HasHelpers::PendingDBOperation.where(enqueued_at: nil).where(is_slow: is_slow).order(:created_at).limit(current_limit).to_sql
+      HasHelpers::PendingDBOperation.uncached do
+        pending_jobs = HasHelpers::PendingDBOperation.find_by_sql(pending_jobs_sql) # The reason behind doing this is to bypass ActiveRecord cache, I tried with just `uncached` but Rails was still caching the result
+        enqueue_jobs(pending_jobs)
+      end
+    end
+
+    def enqueue_jobs(jobs)
+      jobs.each do |job|
+        break if ::HasHelpers::DBOperations::Operations.paused?
+        if job.is_slow
+          enqueue_job_in_redis(job, SLOW_QUEUE) # Enqueue slow jobs directly into the slow queue
+        else
+          enqueue_job_in_redis(job, FAST_QUEUE) # Enqueue regular jobs into the fast queue
+        end
+        job.update!(enqueued_at: Time.zone.now)
+      end
+    end
+
+    # Helper method to enqueue a job into Redis
+    def enqueue_job_in_redis(job, queue_name)
+      redis.zadd(queue_name, job.created_at.to_f, job.id)
+    end
+
+    # Check if the worker for the fast or slow queue is already running by looking at a Redis lock/flag with expiration
+    def worker_running?(lock_key)
+      lock_ttl = redis.ttl(lock_key)
+      lock_ttl > 0 # The lock is active if TTL (Time to Live) is positive
+    end
+
+    def redis
+      RedisConf.current
+    end
+
+    delegate :config, to: :class
+  end
+end

data/app/workers/has_helpers/db_operations/operations.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+# Provides a variety of utilities related to DB operations:
+# 1. Access to a shared logger
+module HasHelpers
+  module DBOperations::Operations
+    MAX_PROCESSING_TIME = 5.minutes
+    PAUSED_KEY = "db_ops_paused"
+    PAUSE_TABLE_KEY_PREFIX = "db_ops_pause_table"
+    DEFAULT_PAUSE_TABLE_DURATION_SECONDS = 1.day.to_i
+
+    def self.run_one_db_operation(id)
+      ::HasHelpers::DBOperations::Operations.with_advisory_lock_2("'pending_db_operations'::regclass::integer", id) do |connection|
+        connection.execute("SELECT run_one_db_operation(#{id})")
+      end
+    end
+
+    def self.enqueue_ops(except_for: nil)
+      pending_db_ops = HasHelpers::PendingDBOperation.where(enqueued_at: nil).order(:created_at).limit(1_000).select(:id)
+      if except_for.present?
+        pending_db_ops = pending_db_ops.where.not(relation_name: except_for)
+      end
+
+      pending_db_ops_sql = pending_db_ops.to_sql
+      Enumerator.new do |yielder|
+        holdstart = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+        loop do
+          enqueued_db_ops = HasHelpers::PendingDBOperation.uncached do
+            HasHelpers::PendingDBOperation.find_by_sql <<-SQL
+              WITH updated_rows AS (
+                UPDATE pending_db_operations
+                SET enqueued_at = now() at time zone 'utc'
+                WHERE id IN (#{pending_db_ops_sql})
+                RETURNING id
+              )
+              SELECT *
+              FROM pending_db_operations
+              INNER JOIN updated_rows ON updated_rows.id = pending_db_operations.id
+            SQL
+          end
+
+          if enqueued_db_ops.present?
+            yielder << enqueued_db_ops
+          else
+            break
+          end
+
+          endt = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+          break if (endt - holdstart) > MAX_PROCESSING_TIME
+        end
+      end
+    end
+
+    def self.defer_processing(id)
+      HasHelpers::PendingDBOperation.defer_processing(id)
+    end
+
+    ## overrides to provide default connection pool
+    def self.with_advisory_lock(key, connection_pool: self.connection_pool, &block)
+      ::PsqlHq::Utils.with_advisory_lock(key, connection_pool: connection_pool, &block)
+    end
+
+    def self.with_advisory_lock_2(key1, key2, connection_pool: self.connection_pool, &block) # rubocop:disable Naming/VariableNumber
+      ::PsqlHq::Utils.with_advisory_lock_2(key1, key2, connection_pool: connection_pool, &block)
+    end
+
+    def self.with_advisory_xact_lock(key, connection_pool: self.connection_pool, &block)
+      ::PsqlHq::Utils.with_advisory_xact_lock(key, connection_pool: connection_pool, &block)
+    end
+
+    def self.with_advisory_xact_lock_2(key1, key2, connection_pool: self.connection_pool, &block) # rubocop:disable Naming/VariableNumber
+      ::PsqlHq::Utils.with_advisory_xact_lock_2(key1, key2, connection_pool: connection_pool, &block)
+    end
+
+    ##### Pause / Unpause #####
+    def self.pause(duration = 24.hours.to_i)
+      ::HasUtils::Env.run_on_environment excluded: [::HasUtils::Env::TEST] do
+        RedisConf.current.setex(PAUSED_KEY, duration, true)
+      end
+    end
+
+    def self.resume
+      RedisConf.current.del(PAUSED_KEY)
+    end
+
+    def self.paused?
+      RedisConf.current.get(PAUSED_KEY).present?
+    end
+
+    # Use the master connection, otherwise this will break when running with crunchy-proxy.
+    # crunchy-proxy can not recongize that a function has writes in it unless you add it to an exception list.
+    def self.connection_pool
+      HasHelpers::MasterRecord.connection_pool
+    end
+
+    ##### Temporary Table Pausing #####
+
+    # Use to indicate that we want to stop processing this table for the
+    # duration indicated. If no duration is specified, defaults to
+    # DEFAULT_PAUSE_TABLE_DURATION_SECONDS. Note that calling this overwrites any
+    # previous call to stop processing this table, even if the previous call's
+    # duration has not expired.
+    def self.pause_table(table_name, duration = DEFAULT_PAUSE_TABLE_DURATION_SECONDS)
+      ::HasUtils::Env.run_on_environment excluded: [::HasUtils::Env::TEST] do
+        RedisConf.current.setex(pause_table_redis_key(table_name), duration, true)
+      end
+    end
+
+    # Returns a boolean indicating if we are pausing processing of this table.
+    def self.table_paused?(table_name)
+      redis_key = pause_table_redis_key(table_name)
+      RedisConf.current.get(redis_key).present?
+    end
+
+    # Cancels any pausing previously set for this table.
+    def self.resume_table(table_name)
+      RedisConf.current.del(pause_table_redis_key(table_name))
+    end
+
+    def self.resume_all_tables
+      keys = RedisConf.current.keys("#{PAUSE_TABLE_KEY_PREFIX}::*")
+      RedisConf.current.del(keys) if keys.present?
+    end
+
+    def self.pause_table_redis_key(table_name)
+      "#{PAUSE_TABLE_KEY_PREFIX}::#{table_name}"
+    end
+
+    def self.disabled_tables
+      RedisConf.current.keys("#{PAUSE_TABLE_KEY_PREFIX}::*").map { |k| k.sub("#{PAUSE_TABLE_KEY_PREFIX}::", "") }
+    end
+  end
+end

data/app/workers/has_helpers/db_operations/update_operations_stats_resume_worker.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  class DBOperations::UpdateOperationsStatsResumeWorker < HasUtils::GoodJob::BaseWorker
+    include HasHelpers::DefaultWorker
+
+    queue_as :db_clean
+
+    def perform
+      ::Rails.logger.info "Truncate db_operations_stats_resumes."
+
+      ActiveRecord::Base.connection.execute <<-END_OF_SQL
+        TRUNCATE db_operations_stats_resumes;
+      END_OF_SQL
+
+      ::Rails.logger.info "Start updating db_operations_stats_resumes"
+
+      ActiveRecord::Base.connection.execute <<-END_OF_SQL
+        INSERT INTO db_operations_stats_resumes
+        SELECT relation_name || (executed_sql_json -> 'sql')::TEXT AS id,
+               relation_name,
+               MAX(created_at)                                     AS last_created_at,
+               SUM(execute_duration)                               AS sum_duration,
+               COUNT(relation_name)                                AS count_calls,
+               executed_sql_json -> 'sql'                          AS executed_sql,
+               MAX(execute_duration)                               AS max_duration
+        FROM db_operations_stats
+        GROUP BY relation_name,
+                 executed_sql_json -> 'sql';
+      END_OF_SQL
+
+      ::Rails.logger.info "Updated db_operations_stats_resumes"
+    end
+  end
+end

data/app/workers/has_helpers/default_worker.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  module DefaultWorker
+    def self.included(base_class)
+      base_class.prepend InstanceMethods
+    end
+
+    module InstanceMethods
+      def perform(...)
+        HasHelpers::MasterRecord.without_pgpool do
+          super
+        end
+      end
+    end
+  end
+end

data/app/workers/has_helpers/good_job/notification_alert_cleanup_worker.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  module GoodJob
+    class NotificationAlertCleanupWorker < HasUtils::GoodJob::BaseWorker
+      queue_as :default
+
+      def perform
+        cleanup_old_alerts
+      end
+
+      private
+
+      def cleanup_old_alerts
+        cutoff_date = 1.month.ago
+
+        Rails.logger.info("Starting cleanup of notification alerts older than #{cutoff_date}")
+
+        old_alerts = HasHelpers::NotificationAlert.where("created_at < ?", cutoff_date)
+        count = old_alerts.count
+
+        if count > 0
+          Rails.logger.info("Found #{count} notification alerts to clean up")
+
+          # batches to avoid memory issues
+          old_alerts.find_in_batches(batch_size: 100) do |batch|
+            batch_ids = batch.map(&:id)
+            HasHelpers::NotificationAlert.where(id: batch_ids).delete_all
+            Rails.logger.info("Deleted batch of #{batch.size} notification alerts")
+          end
+
+          Rails.logger.info("Successfully cleaned up #{count} old notification alerts")
+        else
+          Rails.logger.info("No old notification alerts found to clean up")
+        end
+      rescue StandardError => e
+        Rails.logger.error("Error cleaning up notification alerts: #{e.message}")
+        Rails.logger.error(e.backtrace.join("\n"))
+        raise e
+      end
+    end
+  end
+end

data/app/workers/has_helpers/service_trust/audit_log_cleanup_worker.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  module ServiceTrust
+    class AuditLogCleanupWorker < HasUtils::GoodJob::BaseWorker
+      queue_as :default
+
+      def perform
+        cutoff = Time.current - HasHelpers::ServiceTrust.config.audit_log_retention
+        loop do
+          ids = AuditLog.where("occurred_at < ?", cutoff).limit(10_000).pluck(:id)
+          break if ids.empty?
+
+          AuditLog.where(id: ids).delete_all
+        end
+      end
+    end
+  end
+end