rails_error_dashboard 0.14.4 → 0.16.0

data/lib/rails_error_dashboard/services/missing_translation_tracker.rb

@@ -0,0 +1,316 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    # Counts the HOST app's missing translations in a thread-local buffer and
+    # writes them out at the end of the request or job that produced them.
+    #
+    # WHY A BUFFER: a missing key in a partial rendered inside a loop misses
+    # once per iteration, on every request, for every user. Counting is a hash
+    # increment; the database sees one upsert per (locale, key) per thread per
+    # flush interval, however many times the key missed. Same shape as
+    # RackAttackTracker, for the same reason.
+    #
+    # Misses are fed in by MissingTranslationHandler, which wraps
+    # I18n.exception_handler. RED's own dashboard strings never arrive here:
+    # I18nStore resolves them in a private backend that never reaches the
+    # global handler.
+    #
+    # STORAGE: Thread#thread_variable_get/set, not Thread.current[]. The
+    # latter is fiber-local: a streaming template (render stream: true) runs
+    # in a Fiber, so misses counted there would vanish with the fiber and the
+    # executor hook on the root fiber would find nothing to write. Thread
+    # variables are shared by every fiber of the thread, which is what "per
+    # thread" has to mean for the drain point and the record point to agree.
+    #
+    # SAFETY RULES (HOST_APP_SAFETY.md):
+    # - Zero I/O in the record path: a hash lookup, an increment, and (for a
+    #   key seen for the first time on this thread) one lazy stack scan
+    # - Never raises: every public method rescues
+    # - Thread-local state, no mutex
+    # - Bounded: at most MAX_BUFFERED_KEYS entries per thread; further new keys
+    #   are counted in an overflow bucket rather than dropped silently
+    # - The buffer drains only off the request path (executor to_complete,
+    #   after the response body is closed) and, for threads still alive, at
+    #   process exit
+    class MissingTranslationTracker
+      COUNTS_THREAD_KEY = :red_missing_translation_counts
+
+      # Monotonic time at which the buffer became non-empty -- the deadline
+      # clock, seeded when the first miss lands so a buffer that receives one
+      # miss and then goes quiet is still written out (see RackAttackTracker).
+      DEADLINE_THREAD_KEY = :red_missing_translation_deadline_at
+
+      # Column limits from the migration. Truncation happens here so a value
+      # never reaches the unique upsert index longer than the column allows.
+      MAX_LOCALE_LENGTH = 35
+      MAX_KEY_LENGTH    = 191
+      MAX_SOURCE_LENGTH = 250
+
+      # Distinct (locale, key) pairs one thread will hold before new keys go
+      # to the overflow bucket. A real app has a few dozen missing keys, not
+      # hundreds; the cap exists for the pathological case (a key built from
+      # user input) so memory stays bounded.
+      MAX_BUFFERED_KEYS = 500
+
+      # Max age of buffered misses before the end-of-request drain writes
+      # them. The upper bound on how stale the dashboard page can be, not a
+      # per-request cost: a flood still collapses to one write per thread per
+      # interval.
+      FLUSH_INTERVAL = 5
+
+      OVERFLOW_KEY    = "__overflow__"
+      OVERFLOW_LOCALE = "*"
+
+      # Composite buffer key separator (ASCII unit separator): cannot appear in
+      # a locale and will not appear in a sane translation key.
+      KEY_SEPARATOR = "\x1F"
+
+      # This gem's own code directories, wherever the gem lives and whatever
+      # its directory is called (a `path:` Gemfile entry, vendor/bundle, a
+      # fork). Frames under them are never the host's call site. lib/ and
+      # app/ rather than the whole install root: in this gem's own checkout
+      # the spec suite sits beside them and is the "host" being tested.
+      GEM_ROOTS = [
+        "#{File.expand_path('../..', __dir__)}/",        # <gem>/lib/
+        "#{File.expand_path('../../../app', __dir__)}/"  # <gem>/app/
+      ].freeze
+
+      # Frames from other gems, Ruby itself and its internals are not the
+      # host's call site either. Only consulted for frames outside Rails.root.
+      FOREIGN_FRAME = %r{/gems/|/ruby/\d|<internal:}
+
+      # How deep host_call_site looks before giving up. The host frame of a
+      # view miss sits a handful of frames below the I18n/ActionView ones.
+      MAX_FRAMES_SCANNED = 40
+
+      class << self
+        # Record one miss. Called by MissingTranslationHandler on every
+        # I18n::MissingTranslation that reaches the global handler.
+        #
+        # @param locale [String, Symbol]
+        # @param key [String] the full dotted key as I18n normalised it
+        # @param source [String, nil] call site; resolved from the stack when nil
+        #   and the key is new to this thread's buffer
+        def record(locale:, key:, source: nil)
+          return unless enabled?
+
+          buffer_key = build_key(truncate(locale, MAX_LOCALE_LENGTH), truncate(key, MAX_KEY_LENGTH))
+          counts = counts_for(Thread.current)
+
+          # Seed the deadline the moment the buffer goes from empty to
+          # non-empty, so "never older than FLUSH_INTERVAL" holds for a buffer
+          # that receives exactly one miss.
+          Thread.current.thread_variable_set(DEADLINE_THREAD_KEY, monotonic_now) if counts.empty?
+
+          entry = counts[buffer_key]
+          if entry
+            entry[0] += 1
+          elsif counts.size >= MAX_BUFFERED_KEYS
+            # Bounded memory: the count is kept, the key is not.
+            overflow = (counts[overflow_key] ||= [ 0, nil ])
+            overflow[0] += 1
+          else
+            # The stack is read once per new key per thread, not per miss.
+            counts[buffer_key] = [ 1, truncate(source || host_call_site, MAX_SOURCE_LENGTH) ]
+          end
+
+          nil
+        rescue => e
+          RailsErrorDashboard::Logger.debug(
+            "[RailsErrorDashboard] MissingTranslationTracker.record failed: #{e.class} - #{e.message}"
+          )
+          nil
+        end
+
+        # Write this thread's buffer out, synchronously. Clears the buffer
+        # first so a failed write cannot double-count on the next flush.
+        def flush!
+          drain(Thread.current)
+          nil
+        rescue => e
+          RailsErrorDashboard::Logger.debug(
+            "[RailsErrorDashboard] MissingTranslationTracker.flush! failed: #{e.class} - #{e.message}"
+          )
+          nil
+        end
+
+        # Drain this thread's buffer at the end of a request or job, if it has
+        # been waiting at least FLUSH_INTERVAL. Registered on
+        # Rails.application.executor.to_complete, which fires after the
+        # response body is closed -- the client already has its bytes, so this
+        # never delays a request. Interval-gated so a flood is one write per
+        # thread per interval rather than one per request.
+        def flush_if_due!
+          return unless enabled?
+          return unless flush_due?
+
+          flush!
+        rescue => e
+          RailsErrorDashboard::Logger.debug(
+            "[RailsErrorDashboard] MissingTranslationTracker.flush_if_due! failed: #{e.class} - #{e.message}"
+          )
+          nil
+        end
+
+        # Flush every live thread's buffer, not just the caller's. Called from
+        # at_exit, where the exiting thread's own buffer is usually empty.
+        # Threads that are already gone cannot be drained: Puma joins its
+        # worker threads before at_exit runs, so a worker's last
+        # FLUSH_INTERVAL seconds of misses go with it (measured, same as the
+        # Rack Attack tracker). Rescued per thread so one failing write does
+        # not strand the rest.
+        def flush_all_threads!
+          Thread.list.each do |thread|
+            begin
+              drain(thread)
+            rescue => e
+              RailsErrorDashboard::Logger.debug(
+                "[RailsErrorDashboard] MissingTranslationTracker.flush_all_threads! skipped a thread: #{e.class} - #{e.message}"
+              )
+            end
+          end
+          nil
+        rescue => e
+          RailsErrorDashboard::Logger.debug(
+            "[RailsErrorDashboard] MissingTranslationTracker.flush_all_threads! failed: #{e.class} - #{e.message}"
+          )
+          nil
+        end
+
+        # Clear thread-local state without persisting (specs, thread teardown).
+        def reset!
+          Thread.current.thread_variable_set(COUNTS_THREAD_KEY, nil)
+          Thread.current.thread_variable_set(DEADLINE_THREAD_KEY, nil)
+          nil
+        rescue => e
+          nil
+        end
+
+        # Non-destructive copy of this thread's buffer: { buffer_key => [count, source] }.
+        def buffered_counts
+          (Thread.current.thread_variable_get(COUNTS_THREAD_KEY) || {}).transform_values(&:dup)
+        rescue => e
+          {}
+        end
+
+        # The deadline clock of this thread's buffer, for specs that need to
+        # move it rather than wait out the interval.
+        def deadline
+          Thread.current.thread_variable_get(DEADLINE_THREAD_KEY)
+        end
+
+        def deadline=(value)
+          Thread.current.thread_variable_set(DEADLINE_THREAD_KEY, value)
+        end
+
+        # @return [Array<String>] [locale, translation_key]
+        def parse_key(buffer_key)
+          parts = buffer_key.to_s.split(KEY_SEPARATOR, 2)
+          parts.fill("", parts.length, 2 - parts.length)
+        end
+
+        def overflow_key
+          @overflow_key ||= build_key(OVERFLOW_LOCALE, OVERFLOW_KEY)
+        end
+
+        # True when the buffer has been waiting at least FLUSH_INTERVAL.
+        # Monotonic clock: a wall-clock step backwards would defer the flush
+        # indefinitely.
+        def flush_due?
+          started = deadline
+          return false if started.nil?
+
+          (monotonic_now - started) >= FLUSH_INTERVAL
+        rescue => e
+          false
+        end
+
+        private
+
+        def enabled?
+          RailsErrorDashboard.configuration.enable_missing_translation_tracking
+        rescue => e
+          false
+        end
+
+        def counts_for(thread)
+          thread.thread_variable_get(COUNTS_THREAD_KEY) ||
+            thread.thread_variable_set(COUNTS_THREAD_KEY, {})
+        end
+
+        # Snapshot and clear one thread's buffer, then write the snapshot.
+        def drain(thread)
+          counts = thread.thread_variable_get(COUNTS_THREAD_KEY)
+          return if counts.nil? || counts.empty?
+
+          snapshot = counts.dup
+          counts.clear
+          thread.thread_variable_set(DEADLINE_THREAD_KEY, nil)
+
+          Commands::FlushMissingTranslations.call(counts: snapshot)
+        end
+
+        def build_key(*parts)
+          parts.map(&:to_s).join(KEY_SEPARATOR)
+        end
+
+        def monotonic_now
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        # The host's call site: the first frame under Rails.root, relative to
+        # it ("app/views/users/show.html.erb:12" -- compiled templates keep
+        # the template path and line). Failing that, the first frame that is
+        # neither this gem's nor another gem's nor Ruby's own. Frames are
+        # judged by where they are, never by what the directory is called.
+        #
+        # Thread.each_caller_location (Ruby 3.2+, the gemspec floor) yields
+        # frames one at a time from this method's caller outwards and stops
+        # the moment the block breaks or the method returns.
+        # caller_locations(1, 40) materialised a 40-element array of Location
+        # objects for every new key; the host frame usually sits within the
+        # first ten, so most of that array was allocated to be thrown away.
+        def host_call_site
+          root = rails_root
+          fallback = nil
+          seen = 0
+
+          Thread.each_caller_location do |frame|
+            # Bounded: a pathological stack must not turn into a long scan.
+            break if (seen += 1) > MAX_FRAMES_SCANNED
+
+            path = frame.path.to_s
+            next if path.empty? || GEM_ROOTS.any? { |gem_dir| path.start_with?(gem_dir) }
+
+            if root && path.start_with?(root)
+              return "#{path.delete_prefix(root)}:#{frame.lineno}"
+            end
+
+            fallback ||= "#{path}:#{frame.lineno}" unless path.match?(FOREIGN_FRAME)
+          end
+
+          fallback
+        rescue => e
+          nil
+        end
+
+        def rails_root
+          return nil unless defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+
+          "#{Rails.root}/"
+        rescue => e
+          nil
+        end
+
+        # Truncate to the column limit and strip the separator, so a key that
+        # somehow contained it cannot shift the locale into the key column.
+        def truncate(value, max)
+          s = value.to_s
+          s = s.delete(KEY_SEPARATOR) if s.include?(KEY_SEPARATOR)
+          s.length > max ? s[0, max] : s
+        end
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/repository_url.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    # The one normal form of a configured repository URL, shared by the View
+    # Source links and the issue tracker: no surrounding whitespace, no trailing
+    # slashes, no ".git". The slashes go first, or ".git/" keeps its ".git".
+    module RepositoryUrl
+      # @param url [String, nil]
+      # @return [String, nil]
+      def self.normalize(url)
+        return nil if url.nil?
+
+        url.to_s.strip.sub(%r{/+\z}, "").delete_suffix(".git").sub(%r{/+\z}, "")
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/resolution_callbacks.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    # What happens after an error has been marked resolved, whichever command
+    # did it: the host's `notification_callbacks[:error_resolved]` lambdas
+    # (the issue-tracker integration closes the linked issue from one of
+    # these) and the `error_resolved.rails_error_dashboard` instrumentation
+    # event.
+    #
+    # WHY ONE PLACE: only ResolveError ran these. An error resolved from the
+    # batch toolbar, or through the status dropdown (UpdateErrorStatus),
+    # stayed "resolved" on the dashboard with its GitHub or Linear issue still
+    # open, and nothing subscribed to the event heard about it.
+    #
+    # Plugin hooks are deliberately not here: `:on_error_resolved` and
+    # `:on_errors_batch_resolved` are separate events in the plugin API, and
+    # each command dispatches its own.
+    #
+    # Each callback is rescued on its own, so one failing host lambda does
+    # not stop the next, and never fails the resolution that already happened.
+    class ResolutionCallbacks
+      def self.call(error_log, resolved_by: nil)
+        RailsErrorDashboard.configuration.notification_callbacks[:error_resolved].each do |callback|
+          callback.call(error_log)
+        rescue => e
+          RailsErrorDashboard::Logger.error("Error in error_resolved callback: #{e.message}")
+        end
+
+        ActiveSupport::Notifications.instrument("error_resolved.rails_error_dashboard", {
+          error_log: error_log,
+          error_id: error_log.id,
+          error_type: error_log.error_type,
+          resolved_by: resolved_by,
+          resolved_at: error_log.resolved_at
+        })
+        nil
+      rescue => e
+        RailsErrorDashboard::Logger.error("ResolutionCallbacks failed for error #{error_log&.id}: #{e.message}")
+        nil
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/severity_classifier.rb

@@ -19,7 +19,12 @@ module RailsErrorDashboard
         ActiveRecord::ConnectionNotEstablished
         Redis::ConnectionError
         OpenSSL::SSL::SSLError
+        Zeitwerk::NameError
+        RailsErrorDashboard::TestError
       ].freeze
+      # RailsErrorDashboard::TestError is the Settings page's Send Test Error.
+      # The button promises delivery to every channel, and PagerDuty only
+      # takes critical errors, so the test error is one.
 
       HIGH_SEVERITY_ERROR_TYPES = %w[
         ActiveRecord::RecordNotFound
@@ -67,6 +72,53 @@ module RailsErrorDashboard
       def self.critical?(error_type)
         classify(error_type) == :critical
       end
+
+      BUILT_IN_TYPES = {
+        critical: CRITICAL_ERROR_TYPES,
+        high: HIGH_SEVERITY_ERROR_TYPES,
+        medium: MEDIUM_SEVERITY_ERROR_TYPES
+      }.freeze
+
+      # The error types that classify as +severity+, for a SQL IN filter: the
+      # built-in list for that level, minus any type a custom rule moves
+      # elsewhere, plus any type a custom rule moves here. Filtering on the
+      # bare constants showed CustomPaymentError => :critical under "low" and
+      # a demoted built-in type still under "critical", while every badge on
+      # the page said otherwise.
+      #
+      # :low has no list of its own (it is "everything else"); callers use
+      # `where.not(error_type: categorized_error_types)` for it.
+      #
+      # @param severity [Symbol, String] :critical, :high or :medium
+      # @return [Array<String>]
+      def self.error_types_for(severity)
+        level = severity.to_sym
+        rules = custom_rules
+        built_in = BUILT_IN_TYPES.fetch(level, [])
+
+        (built_in - rules.keys) + rules.select { |_type, sev| sev == level }.keys
+      end
+
+      # Every error type that classifies as something other than :low, so
+      # "low" is `where.not(error_type: categorized_error_types)`.
+      # @return [Array<String>]
+      def self.categorized_error_types
+        rules = custom_rules
+        built_in = BUILT_IN_TYPES.values.flatten
+        promoted = rules.reject { |_type, sev| sev == :low }.keys
+        demoted  = rules.select { |_type, sev| sev == :low }.keys
+
+        (built_in - demoted) | promoted
+      end
+
+      # custom_severity_rules with String keys and Symbol values, whatever the
+      # initializer used.
+      def self.custom_rules
+        (RailsErrorDashboard.configuration.custom_severity_rules || {})
+          .to_h { |type, sev| [ type.to_s, sev.to_s.to_sym ] }
+      rescue StandardError
+        {}
+      end
     end
   end
 end

data/lib/rails_error_dashboard/services/similarity_calculator.rb

@@ -80,19 +80,13 @@ module RailsErrorDashboard
       end
 
       # Extract meaningful frames from backtrace
-      # Format: "file.rb:123:in `method_name`" => "file.rb:method_name"
+      # "file.rb:123:in `name'" (Ruby 3.3) or "file.rb:123:in 'Class#name'"
+      # (Ruby 3.4) => "file.rb:name", via BacktraceProcessor.frame_key
       def extract_frames(backtrace)
         return [] if backtrace.blank?
 
         lines = backtrace.is_a?(String) ? backtrace.split("\n") : backtrace
-        lines.first(20).map do |line|  # Only consider first 20 frames
-          # Extract file path and method name, ignore line numbers
-          if line =~ %r{([^/]+\.rb):.*?in `(.+)'$}
-            "#{Regexp.last_match(1)}:#{Regexp.last_match(2)}"
-          elsif line =~ %r{([^/]+\.rb)}
-            Regexp.last_match(1)
-          end
-        end.compact.uniq
+        lines.first(20).map { |line| BacktraceProcessor.frame_key(line) }.compact.uniq  # Only consider first 20 frames
       end
 
       # Normalize message for comparison

data/lib/rails_error_dashboard/services/source_code_reader.rb

@@ -13,6 +13,14 @@ module RailsErrorDashboard
       MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MB
       MAX_CONTEXT_LINES = 50
 
+      # The files a Ruby backtrace can point to: Ruby source and view templates.
+      # A frame's path is whatever the stored backtrace says, and the files
+      # beside the code (config/*.yml, logs, dotfiles) are where secrets live.
+      RUBY_EXTENSIONS = %w[.rb .rake .ru].freeze
+      # Templates only under an app/ directory (app/views, app/components, an
+      # engine's app/): elsewhere .erb is configuration, as in config/database.yml.erb.
+      TEMPLATE_EXTENSIONS = %w[.erb .haml .slim .builder .jbuilder].freeze
+
       attr_reader :file_path, :line_number, :error
 
       # Initialize a new source code reader
@@ -112,14 +120,26 @@ module RailsErrorDashboard
       def validate_path!(path)
         # Normalize path
         normalized = File.expand_path(path)
-        rails_root = File.expand_path(Rails.root)
+        root_prefix = File.join(File.expand_path(Rails.root), "")
 
-        # Must be within Rails.root
-        unless normalized.start_with?(rails_root)
+        # Must be within Rails.root. The trailing separator matters: without it
+        # a root of /app also admitted /app-old.
+        unless normalized.start_with?(root_prefix)
           RailsErrorDashboard::Logger.warn("Path outside Rails.root: #{normalized}")
           return false
         end
 
+        # Source files only, and nothing under a dot directory (.git, .bundle)
+        segments = normalized.delete_prefix(root_prefix).split("/")
+        extension = File.extname(normalized).downcase
+        source = RUBY_EXTENSIONS.include?(extension) ||
+          (TEMPLATE_EXTENSIONS.include?(extension) && segments[0...-1].include?("app"))
+        hidden = segments.any? { |segment| segment.start_with?(".") }
+        if !source || hidden
+          RailsErrorDashboard::Logger.warn("Not a source file: #{normalized}")
+          return false
+        end
+
         # Prevent directory traversal
         if path.include?("..") || normalized.include?("..")
           RailsErrorDashboard::Logger.warn("Directory traversal attempt: #{path}")

data/lib/rails_error_dashboard/services/storm_protection/count_buffer.rb

@@ -38,7 +38,8 @@ module RailsErrorDashboard
         Entry = Struct.new(
           :error_class, :message, :first_app_frame,
           :controller_name, :action_name, :custom_hash, :environment,
-          :opaque_identity, :count, :first_seen_at, :last_seen_at, :buckets
+          :opaque_identity, :count, :first_seen_at, :last_seen_at, :buckets,
+          :capture_source
         )
 
         # The bucket an instant belongs to, as an epoch second.
@@ -120,6 +121,7 @@ module RailsErrorDashboard
               # job CANNOT recompute this -- it has to travel, or a storm count
               # lands on a different row than the full capture path would.
               "opaque_identity" => entry.opaque_identity,
+              "capture_source" => entry.capture_source,
               "count" => entry.count.value,
               "first_seen_at" => entry.first_seen_at.iso8601,
               "last_seen_at" => entry.last_seen_at.iso8601,
@@ -167,7 +169,7 @@ module RailsErrorDashboard
                 parts[:controller_name], parts[:action_name], parts[:custom_hash], parts[:environment],
                 parts[:opaque_identity],
                 Concurrent::AtomicFixnum.new(0), first_seen_at || Time.current, last_seen_at || Time.current,
-                Concurrent::Map.new
+                Concurrent::Map.new, parts[:capture_source]
               )
             end
           end
@@ -216,7 +218,8 @@ module RailsErrorDashboard
             error_class: entry["error_class"], message: entry["message"],
             first_app_frame: entry["first_app_frame"], controller_name: entry["controller_name"],
             action_name: entry["action_name"], custom_hash: entry["custom_hash"],
-            environment: entry["environment"], opaque_identity: entry["opaque_identity"]
+            environment: entry["environment"], opaque_identity: entry["opaque_identity"],
+            capture_source: entry["capture_source"]
           }
         end
 

data/lib/rails_error_dashboard/services/storm_protection/gate.rb

@@ -279,7 +279,12 @@ module RailsErrorDashboard
               # Only an EXPLICIT environment from the caller. nil means "the
               # worker's own environment", resolved at flush time exactly as
               # LogError resolves it for a full capture.
-              environment: context[:environment].to_s.strip.presence&.[](0, 64)
+              environment: context[:environment].to_s.strip.presence&.[](0, 64),
+              # Whose backtrace first_app_frame came from, for the row the flush
+              # may build from it. Not part of the identity or the gate key: a
+              # client report and a server capture of one error share an entry,
+              # which keeps the first event's frame, and so its source.
+              capture_source: context[:_capture_source] == "client" ? "client" : "server"
             )
           end