rails_pulse 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1a0c55a1386d00790c4bada3da5ed8b34c17e670c7bf1a0c8c55a55c662b16c
-  data.tar.gz: 6de45806d2c72656dcc2ef06e61e00a7dd013e826a69a3f12f49cecf9efd360e
+  metadata.gz: b54f69a078537dd6dabc4074623c3deaf4b0123f4c229b57c706583f104d1b62
+  data.tar.gz: 2cd90650cc8ceb9cf15dac48c4d4695113c73a52fdb4661153fa5c45cbf3b378
 SHA512:
-  metadata.gz: a946a339e72c604b0202f28f2e5650f9034619c1737f308cc14a640331a9d25ed60cfa86fc03c2c5a5bc9ab0793ab661844c93bcd1d394d0eedf7984e2fca211
-  data.tar.gz: ebb59bb9f9c60c0a9b5ce5ab0a778d08396a841cab960b6f4f4a16a41ca86a38e5ab54e3902b949327b84f5bb6618a7de35366993fded467e8a3b6e5e946ddc6
+  metadata.gz: 125303510a989e6de226b732d14fd76d001b5641feb4f93bd8ae5daf07ec73b2328c022097c36be6b4153cd270a9344bf8d4f14e1364843b96fdabd07beb0aab
+  data.tar.gz: d262a5d599007772f1cccc087b6166fd3a151f93f3cdf990bcd37b3ffed6dc6df3aa785a959ae0ae92ac56df545d9a299aed1ec41e3f3d605adb868ce9bb22d0

data/CHANGELOG.md

@@ -5,7 +5,16 @@ All notable changes to Rails Pulse will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## 0.4.2 - 2025-10-01
+
+### Changed
+
+- Patch releases can be cut from a stable branch (`0-4-stable` for 0.4.x) while `main` carries the next release; `bin/release` and CI accept those branches.
+
+### Fixed
+
+- **Cleanup deletes in bounded batches.** A table far over its limit was trimmed with one statement whose size grew with the overage, which could exhaust the database server's memory on a busy app; every delete now removes at most 5,000 rows per statement. (#322)
+- **One failed cleanup stage no longer stops the rest.** The remaining tables are still cleaned and the error is raised once they have finished. (#322)
 
 ## [0.4.1] - 2026-09-23
 

data/lib/rails_pulse/cleanup_service.rb

@@ -1,5 +1,13 @@
 module RailsPulse
   class CleanupService
+    # Rows removed per DELETE statement. Statement size, lock time and
+    # transaction length stay flat however far a table has outgrown its limits.
+    BATCH_SIZE = 5_000
+
+    # Parent rows handled per round when their child rows must go first. The
+    # parent ids are held in Ruby and sent back as a list, so this bounds it.
+    PARENT_BATCH_SIZE = 1_000
+
     def self.perform
       new.perform
     end
@@ -11,6 +19,7 @@ module RailsPulse
         count_based: {},
         total_deleted: 0
       }
+      @failures = []
     end
 
     def perform
@@ -23,11 +32,65 @@ module RailsPulse
       perform_summary_cleanup
 
       log_cleanup_summary
+      raise @failures.first[:error] if @failures.any?
+
       @stats
     end
 
     private
 
+    # Runs one table's cleanup and records its count. A failure is logged and
+    # held until every stage has run, so one table that cannot be cleaned does
+    # not stop the others; each stage removes its own child rows or skips rows
+    # that still have them, so none depends on an earlier stage succeeding.
+    def run_stage(group, name)
+      @stats[group][name] = yield
+    rescue StandardError => e
+      @stats[group][name] = 0
+      @failures << { stage: "#{name} (#{group})", error: e }
+      RailsPulse.logger.error "Cleanup stage #{name} (#{group}) failed: #{e.class}: #{e.message}"
+    end
+
+    # Deletes the relation's rows in statements of at most BATCH_SIZE rows,
+    # choosing each batch inside the database. `order_column` decides which
+    # rows go first; `limit` stops after that many rows.
+    def delete_in_batches(relation, order_column: nil, limit: nil)
+      relation = relation.order(order_column => :asc) if order_column
+      deleted = 0
+
+      loop do
+        batch_size = limit ? [ BATCH_SIZE, limit - deleted ].min : BATCH_SIZE
+        break if batch_size <= 0
+
+        count = relation.limit(batch_size).delete_all
+        deleted += count
+        break if count < batch_size
+      end
+
+      deleted
+    end
+
+    # Deletes parent rows oldest first, removing each batch's child rows before
+    # the parents — there is no ON DELETE CASCADE on the foreign keys. Returns
+    # the number of parent rows deleted.
+    def delete_parents_in_batches(parents, order_column:, children:, foreign_key:, limit: nil)
+      deleted = 0
+
+      loop do
+        batch_size = limit ? [ PARENT_BATCH_SIZE, limit - deleted ].min : PARENT_BATCH_SIZE
+        break if batch_size <= 0
+
+        ids = parents.order(order_column => :asc).limit(batch_size).pluck(:id)
+        break if ids.empty?
+
+        delete_in_batches(children.where(foreign_key => ids))
+        deleted += parents.where(id: ids).delete_all
+        break if ids.size < batch_size
+      end
+
+      deleted
+    end
+
     def cleanup_enabled?
       @config.archiving_enabled
     end
@@ -41,15 +104,15 @@ module RailsPulse
       RailsPulse.logger.info "Time-based cleanup: removing records older than #{cutoff_time}"
 
       # Clean up in order that respects foreign key constraints
-      @stats[:time_based][:operations]            = cleanup_operations_by_time(cutoff_time)
-      @stats[:time_based][:job_runs]              = cleanup_job_runs_by_time(cutoff_time)
-      @stats[:time_based][:requests]              = cleanup_requests_by_time(cutoff_time)
-      @stats[:time_based][:queries]               = cleanup_queries_by_time(cutoff_time)
-      @stats[:time_based][:routes]                = cleanup_routes_by_time(cutoff_time)
-      @stats[:time_based][:jobs]                  = cleanup_jobs_by_time(cutoff_time)
+      run_stage(:time_based, :operations) { cleanup_operations_by_time(cutoff_time) }
+      run_stage(:time_based, :job_runs)   { cleanup_job_runs_by_time(cutoff_time) }
+      run_stage(:time_based, :requests)   { cleanup_requests_by_time(cutoff_time) }
+      run_stage(:time_based, :queries)    { cleanup_queries_by_time(cutoff_time) }
+      run_stage(:time_based, :routes)     { cleanup_routes_by_time(cutoff_time) }
+      run_stage(:time_based, :jobs)       { cleanup_jobs_by_time(cutoff_time) }
       if exception_tables_exist?
-        @stats[:time_based][:exception_occurrences] = cleanup_exception_occurrences_by_time(cutoff_time)
-        @stats[:time_based][:exception_groups]      = cleanup_orphaned_exception_groups
+        run_stage(:time_based, :exception_occurrences) { cleanup_exception_occurrences_by_time(cutoff_time) }
+        run_stage(:time_based, :exception_groups)      { cleanup_orphaned_exception_groups }
       end
     end
 
@@ -58,28 +121,20 @@ module RailsPulse
 
       RailsPulse.logger.info "Count-based cleanup: enforcing table record limits"
 
-      # Only delete records from periods that have already been summarized.
-      # This prevents count-based cleanup from removing data before the summary
-      # job has had a chance to aggregate it.
-      cutoff = summarized_cutoff
-
-      # Operations: only delete those before the summarization cutoff
-      ops_scope = cutoff ? RailsPulse::Operation.where("occurred_at < ?", cutoff) : RailsPulse::Operation.none
-
       # Clean up in order that respects foreign key constraints
-      @stats[:count_based][:operations]            = cleanup_by_count(RailsPulse::Operation, :rails_pulse_operations, order_column: :occurred_at, scope: ops_scope)
-      @stats[:count_based][:job_runs]              = cleanup_job_runs_by_count
-      @stats[:count_based][:requests]              = cleanup_requests_by_count
-      @stats[:count_based][:queries]               = cleanup_queries_by_count
-      @stats[:count_based][:routes]                = cleanup_routes_by_count
-      @stats[:count_based][:jobs]                  = cleanup_jobs_by_count
+      run_stage(:count_based, :operations) { cleanup_operations_by_count }
+      run_stage(:count_based, :job_runs)   { cleanup_job_runs_by_count }
+      run_stage(:count_based, :requests)   { cleanup_requests_by_count }
+      run_stage(:count_based, :queries)    { cleanup_queries_by_count }
+      run_stage(:count_based, :routes)     { cleanup_routes_by_count }
+      run_stage(:count_based, :jobs)       { cleanup_jobs_by_count }
       if exception_tables_exist?
-        @stats[:count_based][:exception_occurrences] = cleanup_exception_occurrences_by_count
-        @stats[:count_based][:exception_groups]      = cleanup_exception_groups_by_count
-        @stats[:count_based][:orphaned_exception_groups] = cleanup_orphaned_exception_groups
+        run_stage(:count_based, :exception_occurrences)     { cleanup_exception_occurrences_by_count }
+        run_stage(:count_based, :exception_groups)          { cleanup_exception_groups_by_count }
+        run_stage(:count_based, :orphaned_exception_groups) { cleanup_orphaned_exception_groups }
       end
       if deployments_table_exists?
-        @stats[:count_based][:deployments] = cleanup_deployments_by_count
+        run_stage(:count_based, :deployments) { cleanup_deployments_by_count }
       end
     end
 
@@ -101,56 +156,77 @@ module RailsPulse
       return 0 unless max_records
 
       relation = scope || model_class
-      current_count = relation.count
-      return 0 if current_count <= max_records
+      overage = relation.count - max_records
+      return 0 if overage <= 0
 
-      records_to_delete = current_count - max_records
-      ids_to_delete = relation.order(order_column => :asc).limit(records_to_delete).pluck(:id)
-      model_class.where(id: ids_to_delete).delete_all
-      records_to_delete
+      delete_in_batches(relation, order_column: order_column, limit: overage)
     end
 
     # Time-based cleanup methods
 
     def cleanup_operations_by_time(cutoff_time)
-      RailsPulse::Operation.where("occurred_at < ?", cutoff_time).delete_all
+      delete_in_batches(RailsPulse::Operation.where("occurred_at < ?", cutoff_time), order_column: :occurred_at)
     end
 
     def cleanup_requests_by_time(cutoff_time)
-      request_subquery = RailsPulse::Request.where("occurred_at < ?", cutoff_time).select(:id)
-      RailsPulse::Operation.where(request_id: request_subquery).delete_all
-      RailsPulse::Request.where("occurred_at < ?", cutoff_time).delete_all
+      delete_parents_in_batches(
+        RailsPulse::Request.where("occurred_at < ?", cutoff_time),
+        order_column: :occurred_at,
+        children: RailsPulse::Operation,
+        foreign_key: :request_id
+      )
     end
 
     def cleanup_queries_by_time(cutoff_time)
-      RailsPulse::Query
-        .where("created_at < ?", cutoff_time)
-        .where("NOT EXISTS (SELECT 1 FROM rails_pulse_operations WHERE rails_pulse_operations.query_id = rails_pulse_queries.id)")
-        .delete_all
+      delete_in_batches(
+        RailsPulse::Query
+          .where("created_at < ?", cutoff_time)
+          .where("NOT EXISTS (SELECT 1 FROM rails_pulse_operations WHERE rails_pulse_operations.query_id = rails_pulse_queries.id)")
+      )
     end
 
     def cleanup_routes_by_time(cutoff_time)
-      RailsPulse::Route
-        .where("created_at < ?", cutoff_time)
-        .where("NOT EXISTS (SELECT 1 FROM rails_pulse_requests WHERE rails_pulse_requests.route_id = rails_pulse_routes.id)")
-        .delete_all
+      delete_in_batches(
+        RailsPulse::Route
+          .where("created_at < ?", cutoff_time)
+          .where("NOT EXISTS (SELECT 1 FROM rails_pulse_requests WHERE rails_pulse_requests.route_id = rails_pulse_routes.id)")
+      )
     end
 
     def cleanup_job_runs_by_time(cutoff_time)
-      job_run_subquery = RailsPulse::JobRun.where("occurred_at < ?", cutoff_time).select(:id)
-      RailsPulse::Operation.where(job_run_id: job_run_subquery).delete_all
-      RailsPulse::JobRun.where("occurred_at < ?", cutoff_time).delete_all
+      delete_parents_in_batches(
+        RailsPulse::JobRun.where("occurred_at < ?", cutoff_time),
+        order_column: :occurred_at,
+        children: RailsPulse::Operation,
+        foreign_key: :job_run_id
+      )
     end
 
     def cleanup_jobs_by_time(cutoff_time)
-      RailsPulse::Job
-        .where("created_at < ?", cutoff_time)
-        .where("NOT EXISTS (SELECT 1 FROM rails_pulse_job_runs WHERE rails_pulse_job_runs.job_id = rails_pulse_jobs.id)")
-        .delete_all
+      delete_in_batches(
+        RailsPulse::Job
+          .where("created_at < ?", cutoff_time)
+          .where("NOT EXISTS (SELECT 1 FROM rails_pulse_job_runs WHERE rails_pulse_job_runs.job_id = rails_pulse_jobs.id)")
+      )
     end
 
     # Count-based cleanup methods (complex cases that need custom scoping)
 
+    # Only operations from periods that have already been summarized are
+    # deleted, so count-based cleanup never removes data the summary job has
+    # not yet aggregated.
+    def cleanup_operations_by_count
+      cutoff = summarized_cutoff
+      return 0 unless cutoff
+
+      cleanup_by_count(
+        RailsPulse::Operation,
+        :rails_pulse_operations,
+        order_column: :occurred_at,
+        scope: RailsPulse::Operation.where("occurred_at < ?", cutoff)
+      )
+    end
+
     def cleanup_requests_by_count
       max_records = @config.max_table_records[:rails_pulse_requests]
       return 0 unless max_records
@@ -159,36 +235,32 @@ module RailsPulse
       cutoff = summarized_cutoff
       return 0 unless cutoff
 
-      current_count = RailsPulse::Request.count
-      return 0 if current_count <= max_records
-
-      records_to_delete = current_count - max_records
-      ids_to_delete = RailsPulse::Request
-        .where("occurred_at < ?", cutoff)
-        .order(occurred_at: :asc)
-        .limit(records_to_delete)
-        .pluck(:id)
-      return 0 if ids_to_delete.empty?
+      overage = RailsPulse::Request.count - max_records
+      return 0 if overage <= 0
 
-      RailsPulse::Operation.where(request_id: ids_to_delete).delete_all
-      RailsPulse::Request.where(id: ids_to_delete).delete_all
-      ids_to_delete.size
+      delete_parents_in_batches(
+        RailsPulse::Request.where("occurred_at < ?", cutoff),
+        order_column: :occurred_at,
+        children: RailsPulse::Operation,
+        foreign_key: :request_id,
+        limit: overage
+      )
     end
 
     def cleanup_job_runs_by_count
       max_records = @config.max_table_records[:rails_pulse_job_runs]
       return 0 unless max_records
 
-      current_count = RailsPulse::JobRun.count
-      return 0 if current_count <= max_records
-
-      records_to_delete = current_count - max_records
-      ids_to_delete = RailsPulse::JobRun.order(occurred_at: :asc).limit(records_to_delete).pluck(:id)
-      return 0 if ids_to_delete.empty?
+      overage = RailsPulse::JobRun.count - max_records
+      return 0 if overage <= 0
 
-      RailsPulse::Operation.where(job_run_id: ids_to_delete).delete_all
-      RailsPulse::JobRun.where(id: ids_to_delete).delete_all
-      ids_to_delete.size
+      delete_parents_in_batches(
+        RailsPulse::JobRun,
+        order_column: :occurred_at,
+        children: RailsPulse::Operation,
+        foreign_key: :job_run_id,
+        limit: overage
+      )
     end
 
     def cleanup_queries_by_count
@@ -219,7 +291,10 @@ module RailsPulse
     end
 
     def cleanup_exception_occurrences_by_time(cutoff_time)
-      exception_occurrences_cleanup_scope.where("occurred_at < ?", cutoff_time).delete_all
+      delete_in_batches(
+        exception_occurrences_cleanup_scope.where("occurred_at < ?", cutoff_time),
+        order_column: :occurred_at
+      )
     end
 
     def cleanup_exception_occurrences_by_count
@@ -255,29 +330,33 @@ module RailsPulse
       records_to_delete = [ overage, deletable_count ].min
       return 0 if records_to_delete <= 0
 
-      ids_to_delete = scope.order(last_seen_at: :asc).limit(records_to_delete).pluck(:id)
-      return 0 if ids_to_delete.empty?
-
-      RailsPulse::ExceptionOccurrence.where(exception_group_id: ids_to_delete).delete_all
-      RailsPulse::ExceptionGroup.where(id: ids_to_delete).delete_all
+      deleted = delete_parents_in_batches(
+        scope,
+        order_column: :last_seen_at,
+        children: RailsPulse::ExceptionOccurrence,
+        foreign_key: :exception_group_id,
+        limit: records_to_delete
+      )
 
       if overage > deletable_count
         RailsPulse.logger.warn("[RailsPulse] Exception group cap #{max_records} cannot be met: " \
-          "#{current_count - ids_to_delete.size} remain (#{current_count - deletable_count} are preserved/ignored)")
+          "#{current_count - deleted} remain (#{current_count - deletable_count} are preserved/ignored)")
       end
 
-      ids_to_delete.size
+      deleted
     end
 
-    # Delete groups whose last occurrence was removed — uses an atomic subquery
-    # so no orphan can be deleted while a concurrent occurrence is being inserted.
+    # Delete groups whose last occurrence was removed — the orphan check and
+    # the delete are one statement per batch, so no orphan can be deleted while
+    # a concurrent occurrence is being inserted.
     # preserve: true and ignored groups are never deleted automatically.
     def cleanup_orphaned_exception_groups
-      RailsPulse::ExceptionGroup
-        .where(preserve: false)
-        .where.not(status: "ignored")
-        .where("NOT EXISTS (SELECT 1 FROM rails_pulse_exception_occurrences WHERE rails_pulse_exception_occurrences.exception_group_id = rails_pulse_exception_groups.id)")
-        .delete_all
+      delete_in_batches(
+        RailsPulse::ExceptionGroup
+          .where(preserve: false)
+          .where.not(status: "ignored")
+          .where("NOT EXISTS (SELECT 1 FROM rails_pulse_exception_occurrences WHERE rails_pulse_exception_occurrences.exception_group_id = rails_pulse_exception_groups.id)")
+      )
     end
 
     def perform_summary_cleanup
@@ -287,8 +366,12 @@ module RailsPulse
       # buys hour-accurate change points further back — at the cost of summary
       # table growth. Day, week and month summaries are never pruned.
       cutoff = @config.hourly_summary_retention.ago
-      deleted = RailsPulse::Summary.where(period_type: "hour").where("period_start < ?", cutoff).delete_all
-      @stats[:time_based][:hourly_summaries] = deleted
+      run_stage(:time_based, :hourly_summaries) do
+        delete_in_batches(
+          RailsPulse::Summary.where(period_type: "hour").where("period_start < ?", cutoff),
+          order_column: :period_start
+        )
+      end
     end
 
     # Returns the period_end of the most recent completed hourly overall-request
@@ -305,6 +388,10 @@ module RailsPulse
       total_count_based = @stats[:count_based].values.sum
       @stats[:total_deleted] = total_time_based + total_count_based
 
+      if @failures.any?
+        RailsPulse.logger.error "Cleanup finished with #{@failures.size} failed stage(s): #{@failures.map { |failure| failure[:stage] }.join(', ')}"
+      end
+
       RailsPulse.logger.info "Cleanup completed:"
       RailsPulse.logger.info "  Time-based: #{total_time_based} records deleted"
       RailsPulse.logger.info "  Count-based: #{total_count_based} records deleted"

data/lib/rails_pulse/version.rb

@@ -1,3 +1,3 @@
 module RailsPulse
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails_pulse
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Scott Harvey
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-09-23 00:00:00.000000000 Z
+date: 2026-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -551,7 +551,7 @@ metadata:
   bug_tracker_uri: https://github.com/railspulse/rails_pulse/issues
   rubygems_mfa_required: 'true'
 post_install_message: |
-  Rails Pulse 0.4.1 installed.
+  Rails Pulse 0.4.2 installed.
 
   UPGRADING from any 0.3.x? This release changes route identity and DROPS
   rails_pulse_routes.method. Back up your database first, then run: