rails_error_dashboard 0.12.0 → 0.13.0

data/lib/rails_error_dashboard/services/swallowed_exception_tracker.rb

@@ -25,6 +25,17 @@ module RailsErrorDashboard
       RAISE_THREAD_KEY  = :red_swallowed_raises
       RESCUE_THREAD_KEY = :red_swallowed_rescues
       FLUSH_THREAD_KEY  = :red_swallowed_last_flush
+      # Monotonic time at which this thread's buffer stopped being empty. The
+      # flush is due one interval after THAT, whether or not anything else
+      # happens on the thread.
+      DEADLINE_THREAD_KEY = :red_swallowed_armed_at
+
+      # Where evicted counts go, so eviction bounds memory without losing the
+      # total. The flush command splits keys on "|" and "->", so these persist as
+      # ordinary rows under the class name "[overflow]".
+      OVERFLOW_LABEL      = "[overflow]"
+      RAISE_OVERFLOW_KEY  = "#{OVERFLOW_LABEL}|#{OVERFLOW_LABEL}".freeze
+      RESCUE_OVERFLOW_KEY = "#{OVERFLOW_LABEL}|#{OVERFLOW_LABEL}->#{OVERFLOW_LABEL}".freeze
       RAISE_LOC_IVAR    = :@_red_raise_loc
 
       # Flow-control exceptions that are commonly raised/rescued in normal Rails operation.
@@ -95,7 +106,7 @@ module RailsErrorDashboard
         end
 
         # Force flush the current thread's counters (used by job and tests)
-        def flush!
+        def flush!(sync: false)
           raises = Thread.current[RAISE_THREAD_KEY]
           rescues = Thread.current[RESCUE_THREAD_KEY]
           return if raises.nil? && rescues.nil?
@@ -107,14 +118,42 @@ module RailsErrorDashboard
           raises&.clear
           rescues&.clear
           Thread.current[FLUSH_THREAD_KEY] = Time.now.to_f
+          Thread.current[DEADLINE_THREAD_KEY] = nil
 
-          dispatch_flush(raise_snapshot, rescue_snapshot)
+          dispatch_flush(raise_snapshot, rescue_snapshot, sync: sync)
         rescue => e
           RailsErrorDashboard::Logger.debug(
             "[RailsErrorDashboard] SwallowedExceptionTracker.flush! failed: #{e.class} - #{e.message}"
           )
         end
 
+        # Drain this thread's buffer at the end of a unit of work (a request or a
+        # job) once it has been waiting for flush_interval.
+        #
+        # Without this the ONLY in-process drain was maybe_flush! inside the
+        # :rescue callback, so a buffer could only ever be flushed by a LATER
+        # rescue on the SAME thread: a swallowed exception that happened once
+        # stayed invisible until the process exited, and counts on a Puma thread
+        # that retired were lost. Same defect, same fix, as
+        # RackAttackTracker#flush_if_due!.
+        #
+        # Wired to Rails.application.executor.to_complete, which fires after the
+        # response body is closed, so this never delays a request. Deadline-gated:
+        # the usual cost is one thread-local read.
+        def flush_if_due!
+          return unless flush_due?
+
+          # sync: the response is already sent, and one upsert per interval is
+          # cheaper than enqueueing a job to do it.
+          flush!(sync: true)
+          nil
+        rescue => e
+          RailsErrorDashboard::Logger.debug(
+            "[RailsErrorDashboard] SwallowedExceptionTracker.flush_if_due! failed: #{e.class} - #{e.message}"
+          )
+          nil
+        end
+
         # Read current thread's counters (for testing/inspection)
         def current_raises
           Thread.current[RAISE_THREAD_KEY] || {}
@@ -129,6 +168,7 @@ module RailsErrorDashboard
           Thread.current[RAISE_THREAD_KEY] = nil
           Thread.current[RESCUE_THREAD_KEY] = nil
           Thread.current[FLUSH_THREAD_KEY] = nil
+          Thread.current[DEADLINE_THREAD_KEY] = nil
         end
 
         private
@@ -152,11 +192,9 @@ module RailsErrorDashboard
           class_name = exception.class.name || exception.class.to_s
           key = "#{class_name}|#{location}"
 
-          raises = (Thread.current[RAISE_THREAD_KEY] ||= {})
-          raises[key] = (raises[key] || 0) + 1
-
-          # 5. LRU eviction if over capacity
-          evict_oldest!(raises) if raises.size > max_cache_size
+          # 4b/5. Count it, most-recent-last, evicting into the overflow bucket
+          increment!((Thread.current[RAISE_THREAD_KEY] ||= {}), key, RAISE_OVERFLOW_KEY)
+          arm_deadline!
         end
 
         # TracePoint(:rescue) callback
@@ -181,11 +219,9 @@ module RailsErrorDashboard
           class_name = exception.class.name || exception.class.to_s
           key = "#{class_name}|#{raise_loc}->#{rescue_loc}"
 
-          rescues = (Thread.current[RESCUE_THREAD_KEY] ||= {})
-          rescues[key] = (rescues[key] || 0) + 1
-
-          # 5. LRU eviction if over capacity
-          evict_oldest!(rescues) if rescues.size > max_cache_size
+          # 4b/5. Count it, most-recent-last, evicting into the overflow bucket
+          increment!((Thread.current[RESCUE_THREAD_KEY] ||= {}), key, RESCUE_OVERFLOW_KEY)
+          arm_deadline!
 
           # 6. Maybe flush
           maybe_flush!
@@ -210,21 +246,60 @@ module RailsErrorDashboard
           false
         end
 
-        # LRU eviction: delete the oldest key (Ruby hashes maintain insertion order)
-        def evict_oldest!(hash)
-          oldest_key = hash.each_key.first
-          hash.delete(oldest_key) if oldest_key
+        # Count one event. delete-and-reinsert moves the key to the END of the
+        # Hash, so insertion order IS recency order and "oldest" below means least
+        # recently UPDATED. A plain `hash[key] += 1` leaves a key where it was
+        # first inserted, which made the busiest key in the process the first one
+        # evicted.
+        def increment!(hash, key, overflow_key)
+          hash[key] = hash.delete(key).to_i + 1
+
+          # Loops because the overflow bucket takes a slot of its own once it
+          # exists. evict_oldest! returns false when only that bucket is left,
+          # which terminates the loop even with max_cache_size misconfigured to 0.
+          while hash.size > max_cache_size
+            break unless evict_oldest!(hash, overflow_key)
+          end
         end
 
-        # Cheap periodic flush check
-        def maybe_flush!
-          now = Time.now.to_f
-          last_flush = Thread.current[FLUSH_THREAD_KEY] ||= now
-          interval = RailsErrorDashboard.configuration.swallowed_exception_flush_interval
+        # Evict the least recently updated key, folding its count into the
+        # overflow bucket so the total is conserved. The overflow bucket itself is
+        # never the victim. Mirrors RackAttackTracker#evict_oldest!.
+        # @return [Boolean] false once only the overflow bucket remains
+        def evict_oldest!(hash, overflow_key)
+          oldest_key = hash.each_key.find { |k| k != overflow_key }
+          return false unless oldest_key
 
-          return unless (now - last_flush) >= interval
+          dropped = hash.delete(oldest_key).to_i
+          hash[overflow_key] = hash[overflow_key].to_i + dropped if dropped.positive?
+          true
+        end
 
-          flush!
+        # Start the flush clock when the buffer becomes non-empty; a later event
+        # never pushes it out. The old guard (`last_flush ||= now`) was first
+        # evaluated by the SECOND event on a thread, so a lone event had no
+        # deadline at all.
+        def arm_deadline!
+          Thread.current[DEADLINE_THREAD_KEY] ||= monotonic_now
+        end
+
+        # Monotonic: Time.now can jump backwards and would defer the flush.
+        def flush_due?
+          armed_at = Thread.current[DEADLINE_THREAD_KEY]
+          return false if armed_at.nil?
+
+          (monotonic_now - armed_at) >= RailsErrorDashboard.configuration.swallowed_exception_flush_interval.to_f
+        rescue => e
+          false
+        end
+
+        def monotonic_now
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        # Cheap periodic flush check on the :rescue path
+        def maybe_flush!
+          flush! if flush_due?
         end
 
         # Dispatch flush asynchronously via background job (zero I/O in request path).
@@ -263,6 +338,7 @@ module RailsErrorDashboard
             thread[RAISE_THREAD_KEY] = nil
             thread[RESCUE_THREAD_KEY] = nil
             thread[FLUSH_THREAD_KEY] = nil
+            thread[DEADLINE_THREAD_KEY] = nil
 
             dispatch_flush(raise_snapshot, rescue_snapshot, sync: true)
           end

data/lib/rails_error_dashboard/services/url_safety.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module RailsErrorDashboard
+  module Services
+    # Pure algorithm: decide whether a string is safe to use as a link target
+    #
+    # A URL that ends up in an `href` must be absolute http(s). Rails' `link_to`
+    # and ERB escaping stop attribute breakout, but neither rejects a scheme such
+    # as `javascript:` — that has to be an allowlist, and it has to be the same
+    # rule where the value is written and where it is rendered.
+    #
+    # The value is judged exactly as given (no stripping), so what passes here is
+    # byte-for-byte what the caller goes on to store or render.
+    #
+    # @example
+    #   UrlSafety.http_url?("https://github.com/a/b/issues/1") # => true
+    #   UrlSafety.http_url?("javascript:alert(1)")             # => false
+    class UrlSafety
+      # Browsers strip TAB/CR/LF from inside a URL before resolving the scheme,
+      # so "java\tscript:" is live. No legitimate URL contains these or a space.
+      UNSAFE_CHARACTERS = /[[:cntrl:][:space:]]/
+
+      # @param value [String, nil]
+      # @return [Boolean] true only for an absolute http:// or https:// URL with a host
+      def self.http_url?(value)
+        return false unless value.is_a?(String)
+        return false if value.empty? || value.match?(UNSAFE_CHARACTERS)
+
+        uri = URI.parse(value)
+        # URI::HTTPS is a subclass of URI::HTTP
+        uri.is_a?(URI::HTTP) && uri.host.present?
+      rescue URI::InvalidURIError, ArgumentError, EncodingError
+        # ArgumentError/EncodingError: a string with invalid bytes cannot be matched
+        false
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/subscribers/issue_tracker_subscriber.rb

@@ -78,11 +78,20 @@ module RailsErrorDashboard
 
           if existing
             # Link this record to the existing issue instead of creating a new one
-            error_log.update_columns(
+            link = {
               external_issue_url: existing.external_issue_url,
               external_issue_number: existing.external_issue_number,
               external_issue_provider: existing.external_issue_provider
-            )
+            }
+            # The repository is half the issue's identity. Without it
+            # IssueTrackerClient.for_error falls back to the CONFIGURED repo,
+            # and every later comment/close/reopen for this row would hit
+            # whatever issue has that number there. Same column guard as
+            # LinkExistingIssue.
+            if ErrorLog.column_names.include?("external_issue_repo")
+              link[:external_issue_repo] = existing.external_issue_repo
+            end
+            error_log.update_columns(link)
             return false
           end
 

data/lib/rails_error_dashboard/value_objects/error_context.rb

@@ -125,7 +125,9 @@ module RailsErrorDashboard
         # has :request_params as a JSON string but no :request object.
         return @context[:request_params] if params.empty? && @context[:request_params].present?
 
-        params.to_json
+        # Params read off a live request or job object never passed through the
+        # context scrub LogError does, and to_json raises on an invalid byte.
+        Services::EncodingSanitizer.scrub_deep(params).to_json
       end
 
       def extract_user_agent

data/lib/rails_error_dashboard/version.rb

@@ -1,3 +1,3 @@
 module RailsErrorDashboard
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 end

data/lib/rails_error_dashboard.rb

@@ -56,11 +56,14 @@ require "rails_error_dashboard/services/pearson_correlation"
 require "rails_error_dashboard/services/statistical_classifier"
 require "rails_error_dashboard/services/source_code_reader"
 require "rails_error_dashboard/services/git_blame_reader"
+require "rails_error_dashboard/services/git_head_reader"
 require "rails_error_dashboard/services/github_link_generator"
 require "rails_error_dashboard/services/cause_chain_extractor"
 require "rails_error_dashboard/services/environment_snapshot"
 require "rails_error_dashboard/services/system_health_snapshot"
+require "rails_error_dashboard/services/encoding_sanitizer"
 require "rails_error_dashboard/services/sensitive_data_filter"
+require "rails_error_dashboard/services/url_safety"
 require "rails_error_dashboard/services/notification_throttler"
 require "rails_error_dashboard/services/breadcrumb_collector"
 require "rails_error_dashboard/services/n_plus_one_detector"
@@ -132,6 +135,8 @@ require "rails_error_dashboard/commands/upsert_cascade_pattern"
 require "rails_error_dashboard/commands/upsert_baseline"
 require "rails_error_dashboard/commands/flush_swallowed_exceptions"
 require "rails_error_dashboard/commands/flush_rack_attack_events"
+require "rails_error_dashboard/commands/scrub_invalid_encoding"
+require "rails_error_dashboard/commands/backfill_resolved_at"
 require "rails_error_dashboard/queries/errors_list"
 require "rails_error_dashboard/queries/dashboard_stats"
 require "rails_error_dashboard/queries/analytics_stats"
@@ -177,6 +182,32 @@ module RailsErrorDashboard
     end
   end
 
+  # The commit this process is running, read from .git once and remembered.
+  #
+  # LogError stamps every error with a SHA. When neither config.git_sha nor a
+  # platform ENV variable supplies one it used to run `git rev-parse` in a
+  # subprocess PER CAPTURED ERROR. The answer cannot change while the process
+  # lives, so it is resolved once (the engine does it at boot, so normally
+  # not even the first capture pays for it) and nil is remembered too: an app
+  # with no repository must not go looking again for every error.
+  #
+  # @return [String, nil] short SHA
+  def self.detected_git_sha
+    return @detected_git_sha if defined?(@detected_git_sha)
+
+    @detected_git_sha = begin
+      Services::GitHeadReader.call(defined?(Rails) && Rails.respond_to?(:root) ? Rails.root : nil)
+    rescue => e
+      RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] Could not detect git SHA: #{e.class}: #{e.message}")
+      nil
+    end
+  end
+
+  # Forget the memo (specs).
+  def self.reset_detected_git_sha!
+    remove_instance_variable(:@detected_git_sha) if defined?(@detected_git_sha)
+  end
+
   # Register a plugin
   # @param plugin [Plugin] The plugin instance to register
   # @return [Boolean] True if registered successfully, false otherwise

data/lib/tasks/error_dashboard.rake

@@ -459,6 +459,8 @@ namespace :error_dashboard do
     puts "\nDeleting errors..."
     start_time = Time.current
     deleted = scope.delete_all
+    # delete_all skips callbacks, and the stat cards are cached.
+    RailsErrorDashboard::Services::AnalyticsCacheManager.clear
     elapsed = (Time.current - start_time).round(2)
 
     puts "\n✓ Cleanup complete!"
@@ -492,7 +494,7 @@ namespace :error_dashboard do
       application_id: app.id,
       dump_data: dump.to_json,
       captured_at: Time.current,
-      note: ENV["NOTE"]
+      note: RailsErrorDashboard::Services::EncodingSanitizer.scrub(ENV["NOTE"])
     )
 
     puts "\n" + JSON.pretty_generate(dump)
@@ -502,7 +504,7 @@ namespace :error_dashboard do
     puts "\n" + "=" * 70 + "\n"
   end
 
-  desc "Run retention cleanup (delete errors older than retention_days)"
+  desc "Run retention cleanup (delete errors not seen for retention_days)"
   task retention_cleanup: :environment do
     config = RailsErrorDashboard.configuration
 
@@ -517,14 +519,15 @@ namespace :error_dashboard do
     end
 
     cutoff = config.retention_days.days.ago
-    count = RailsErrorDashboard::ErrorLog.where("occurred_at < ?", cutoff).count
+    # The job's own selection, so the number shown is the number deleted.
+    count = RailsErrorDashboard::RetentionCleanupJob.expired_scope(cutoff).count
 
     puts "\n  Retention policy: #{config.retention_days} days"
     puts "  Cutoff date: #{cutoff.strftime('%Y-%m-%d %H:%M:%S')}"
     puts "  Errors to delete: #{count}"
 
     if count.zero?
-      puts "\n  No errors older than #{config.retention_days} days"
+      puts "\n  No errors unseen for more than #{config.retention_days} days"
       puts "\n" + "=" * 80 + "\n"
       next
     end
@@ -550,6 +553,53 @@ namespace :error_dashboard do
     puts "\n" + "=" * 80 + "\n"
   end
 
+  desc "Repair rows stored with invalid UTF-8 or NUL bytes (run once after upgrading on SQLite/MySQL)"
+  task scrub_invalid_encoding: :environment do
+    puts "Scanning error logs and occurrences for invalid bytes..."
+    result = RailsErrorDashboard::Commands::ScrubInvalidEncoding.call
+
+    puts "  scanned:    #{result[:scanned]}"
+    puts "  repaired:   #{result[:repaired]}"
+    puts "  unreadable: #{result[:unreadable].size}"
+    result[:unreadable].each { |row| puts "    - #{row}" }
+  end
+
+  desc "Backfill resolved_at for errors resolved through the status workflow (run once after upgrading)"
+  task backfill_resolved_at: :environment do
+    result = RailsErrorDashboard::Commands::BackfillResolvedAt.call
+
+    puts "Backfilled resolved_at on #{result[:updated]} resolved error(s)."
+    puts "MTTR now includes them, so expect the figure to rise to its true value." if result[:updated].positive?
+  end
+
+  desc "Replace raw session IDs stored on occurrences with keyed digests (idempotent)"
+  task digest_session_ids: :environment do
+    filter = RailsErrorDashboard::Services::SensitiveDataFilter
+    prefix = filter::SESSION_DIGEST_PREFIX
+    scope = RailsErrorDashboard::ErrorOccurrence
+      .where.not(session_id: [ nil, "" ])
+      .where("session_id NOT LIKE ? OR LENGTH(session_id) <> 35", "#{prefix}%")
+
+    digested = 0
+    failed = 0
+    scope.in_batches(of: 1000) do |batch|
+      batch.pluck(:id, :session_id).each do |id, raw|
+        digest = filter.digest_session_id(raw)
+        next if digest == raw # already a digest that the SQL prefilter let through
+
+        if digest
+          RailsErrorDashboard::ErrorOccurrence.where(id: id).update_all(session_id: digest)
+          digested += 1
+        else
+          failed += 1
+        end
+      end
+    end
+
+    puts "Session IDs digested: #{digested}"
+    puts "Could not be digested (left unchanged): #{failed}" if failed.positive?
+  end
+
   desc "Send error digest email (PERIOD=daily|weekly, APP_ID=optional)"
   task send_digest: :environment do
     period = ENV.fetch("PERIOD", "daily")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails_error_dashboard
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Anjan Jagirdar
@@ -283,9 +283,11 @@ files:
 - app/jobs/rails_error_dashboard/baseline_calculation_job.rb
 - app/jobs/rails_error_dashboard/close_linked_issue_job.rb
 - app/jobs/rails_error_dashboard/concerns/localized_job.rb
+- app/jobs/rails_error_dashboard/concerns/plain_channel_message.rb
 - app/jobs/rails_error_dashboard/create_issue_job.rb
 - app/jobs/rails_error_dashboard/discord_error_notification_job.rb
 - app/jobs/rails_error_dashboard/email_error_notification_job.rb
+- app/jobs/rails_error_dashboard/notification_burst_summary_job.rb
 - app/jobs/rails_error_dashboard/pagerduty_error_notification_job.rb
 - app/jobs/rails_error_dashboard/rack_attack_flush_job.rb
 - app/jobs/rails_error_dashboard/reopen_linked_issue_job.rb
@@ -417,6 +419,7 @@ files:
 - db/migrate/20260915000002_add_issue_repo_identity_to_error_logs.rb
 - db/migrate/20260915000003_add_context_provenance_to_error_logs.rb
 - db/migrate/20260915000004_create_storm_flush_batches.rb
+- db/migrate/20260917000001_add_last_notified_at_to_error_logs.rb
 - lib/generators/rails_error_dashboard/install/install_generator.rb
 - lib/generators/rails_error_dashboard/install/templates/README
 - lib/generators/rails_error_dashboard/install/templates/initializer.rb
@@ -427,6 +430,7 @@ files:
 - lib/rails_error_dashboard/commands/add_error_comment.rb
 - lib/rails_error_dashboard/commands/assign_error.rb
 - lib/rails_error_dashboard/commands/backfill_environments.rb
+- lib/rails_error_dashboard/commands/backfill_resolved_at.rb
 - lib/rails_error_dashboard/commands/batch_delete_errors.rb
 - lib/rails_error_dashboard/commands/batch_mute_errors.rb
 - lib/rails_error_dashboard/commands/batch_resolve_errors.rb
@@ -443,6 +447,7 @@ files:
 - lib/rails_error_dashboard/commands/log_error.rb
 - lib/rails_error_dashboard/commands/mute_error.rb
 - lib/rails_error_dashboard/commands/resolve_error.rb
+- lib/rails_error_dashboard/commands/scrub_invalid_encoding.rb
 - lib/rails_error_dashboard/commands/snooze_error.rb
 - lib/rails_error_dashboard/commands/unassign_error.rb
 - lib/rails_error_dashboard/commands/unmute_error.rb
@@ -517,6 +522,7 @@ files:
 - lib/rails_error_dashboard/services/diagnostic_dump_generator.rb
 - lib/rails_error_dashboard/services/digest_builder.rb
 - lib/rails_error_dashboard/services/discord_payload_builder.rb
+- lib/rails_error_dashboard/services/encoding_sanitizer.rb
 - lib/rails_error_dashboard/services/environment_snapshot.rb
 - lib/rails_error_dashboard/services/error_broadcaster.rb
 - lib/rails_error_dashboard/services/error_hash_generator.rb
@@ -524,6 +530,7 @@ files:
 - lib/rails_error_dashboard/services/error_notification_dispatcher.rb
 - lib/rails_error_dashboard/services/exception_filter.rb
 - lib/rails_error_dashboard/services/git_blame_reader.rb
+- lib/rails_error_dashboard/services/git_head_reader.rb
 - lib/rails_error_dashboard/services/github_issue_client.rb
 - lib/rails_error_dashboard/services/github_link_generator.rb
 - lib/rails_error_dashboard/services/gitlab_issue_client.rb
@@ -558,6 +565,7 @@ files:
 - lib/rails_error_dashboard/services/storm_protection/gate.rb
 - lib/rails_error_dashboard/services/swallowed_exception_tracker.rb
 - lib/rails_error_dashboard/services/system_health_snapshot.rb
+- lib/rails_error_dashboard/services/url_safety.rb
 - lib/rails_error_dashboard/services/variable_serializer.rb
 - lib/rails_error_dashboard/services/webhook_payload_builder.rb
 - lib/rails_error_dashboard/subscribers/action_cable_subscriber.rb
@@ -585,7 +593,7 @@ metadata:
   funding_uri: https://github.com/sponsors/AnjanJ
 post_install_message: |
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-    RED (Rails Error Dashboard) v0.12.0
+    RED (Rails Error Dashboard) v0.13.0
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
   First install: