rails_pulse 0.4.2 → 0.5.0.pre.2

data/app/services/rails_pulse/sql_query_normalizer.rb

@@ -1,22 +1,54 @@
 module RailsPulse
   class SqlQueryNormalizer
-    # Above this length, skip the regex-based literal scan in favor of a
-    # slower-per-char but backtracking-free manual scan. Regex matching here
-    # is linear (benchmarked at ~0.4s for an 8,000,000-char literal under a
-    # 1s Regexp.timeout), but a host app can configure Regexp.timeout far
-    # below that, and very long queries are rare enough that the fast path
-    # isn't worth the risk. 100_000 chars leaves generous headroom below
-    # where even an aggressive timeout config could plausibly trip — see
-    # issue #286.
-    LONG_QUERY_THRESHOLD = 100_000
+    # MySQL adapters. MySQL's quoting differs from PostgreSQL's and SQLite's
+    # in two ways the scanner must know about. Its drivers escape a quote
+    # inside a string with a backslash ('O\'Brien', "O\"Brien") rather than
+    # by doubling it, whereas on the other two a backslash in a plain string
+    # is an ordinary character ('C:\' is a complete string). And double
+    # quotes delimit a string, not an identifier (Rails quotes MySQL
+    # identifiers with backticks), so every double-quoted span is a value to
+    # redact, where on the other two it is an identifier when it looks like
+    # one (issue #313).
+    MYSQL_ADAPTERS = %w[mysql2 trilogy].freeze
+
+    # Opening delimiter of a PostgreSQL dollar-quoted string: $$ or $tag$.
+    # \G anchors the match at the scan position, so a failed attempt does
+    # not search ahead through the rest of the string.
+    DOLLAR_QUOTE_OPENER = /\G\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$/
+
+    # String-literal prefixes: PostgreSQL escape strings (E'...'), bit,
+    # hex and national strings (B'...', X'...', N'...'), upper or lower case.
+    STRING_PREFIX_CHARS = "EeBbXxNn"
+
+    # A double-quoted token kept as an identifier on PostgreSQL and SQLite:
+    # a plain identifier or a dotted chain of them ("users.email").
+    # Anything else between double quotes — an email, a sentence, a number —
+    # can only be a value (SQLite accepts a double-quoted string in
+    # hand-written SQL), so it must not survive normalization.
+    QUOTED_IDENTIFIER = /\A[A-Za-z_][A-Za-z0-9_$]*(?:\.[A-Za-z_][A-Za-z0-9_$]*)*\z/
+
+    # Longest double-quoted span that is checked against QUOTED_IDENTIFIER.
+    # PostgreSQL caps an identifier at 63 bytes and MySQL at 64, so a dotted
+    # chain of three is well within this; anything longer is a value, and
+    # skipping the regex keeps a long literal from tripping a host's tight
+    # Regexp.timeout (issue #286).
+    MAX_QUOTED_IDENTIFIER_LENGTH = 256
 
     # Smart normalization: preserve table/column names, replace only literal values
-    def self.normalize(query_string)
-      new(query_string).normalize
+    def self.normalize(query_string, adapter: host_adapter)
+      new(query_string, adapter: adapter).normalize
     end
 
-    def initialize(query_string)
+    # The host application's primary database adapter, which decides the
+    # string-escaping rules. A host that also queries a second database on a
+    # different adapter has those queries normalized with the primary's rules.
+    def self.host_adapter
+      @host_adapter ||= ActiveRecord::Base.connection_db_config.adapter.to_s
+    end
+
+    def initialize(query_string, adapter: self.class.host_adapter)
       @query_string = query_string
+      @mysql = MYSQL_ADAPTERS.include?(adapter.to_s.downcase)
     end
 
     def normalize
@@ -25,6 +57,12 @@ module RailsPulse
 
       normalized = @query_string.dup
 
+      # Step 0: Replace every string literal in one left-to-right pass, so a
+      # quote form occurring inside another (an apostrophe in a dollar-quoted
+      # string, $$ inside a plain string) cannot pair across two separate
+      # literals. Double-quoted spans are kept only when they are identifiers.
+      normalized = replace_string_literals(normalized)
+
       # Step 1: Temporarily protect quoted identifiers
       protected_identifiers = protect_identifiers(normalized)
       normalized = protected_identifiers[:normalized]
@@ -44,121 +82,148 @@ module RailsPulse
 
     private
 
+    # One linear scan over the query replacing every single-quoted string
+    # ('...', with '' doubling and, on MySQL, \'), prefixed string (E'...'
+    # and friends, with backslash escapes; U&'...') and dollar-quoted string
+    # ($$...$$, $tag$...$tag$) with "?". A double-quoted span is copied
+    # through when it is an identifier and replaced with "?" when it is a
+    # value; backticked spans are always identifiers and are copied through.
+    # Left to right with no backtracking, like the databases' own lexers,
+    # and no regex runs across literal content, so a host's tight
+    # Regexp.timeout cannot trip on a long literal (see issue #286).
+    def replace_string_literals(query)
+      result = +""
+      i = 0
+      len = query.length
+
+      while i < len
+        char = query[i]
+
+        case char
+        when "'"
+          close = find_closing_quote(query, i + 1, "'", backslash_escapes: @mysql)
+          if close
+            result << "?"
+            i = close + 1
+          else
+            result << query[i..]
+            break
+          end
+        when '"'
+          close = find_closing_quote(query, i + 1, '"', backslash_escapes: @mysql)
+          if close
+            result << (keep_double_quoted?(query[(i + 1)...close]) ? query[i..close] : "?")
+            i = close + 1
+          else
+            result << query[i..]
+            break
+          end
+        when "`"
+          close = find_closing_quote(query, i + 1, "`")
+          if close
+            result << query[i..close]
+            i = close + 1
+          else
+            result << query[i..]
+            break
+          end
+        when "$"
+          if (m = DOLLAR_QUOTE_OPENER.match(query, i))
+            delimiter = m[0]
+            close = query.index(delimiter, i + delimiter.length)
+            if close
+              result << "?"
+              i = close + delimiter.length
+              next
+            end
+          end
+          result << char
+          i += 1
+        else
+          if STRING_PREFIX_CHARS.include?(char) && query[i + 1] == "'" &&
+              (i == 0 || !query[i - 1].match?(/[A-Za-z0-9_$]/))
+            # Only E'...' takes backslash escapes outside MySQL: N'C:\' is a
+            # complete string on PostgreSQL and SQLite.
+            close = find_closing_quote(query, i + 2, "'", backslash_escapes: @mysql || char.casecmp?("e"))
+            if close
+              result << "?"
+              i = close + 1
+              next
+            end
+          elsif char.casecmp?("u") && query[i + 1] == "&" && query[i + 2] == "'" &&
+              (i == 0 || !query[i - 1].match?(/[A-Za-z0-9_$]/))
+            # PostgreSQL Unicode escape string (U&'...'). Its backslash
+            # introduces a code point, not a quote, so only '' escapes a quote.
+            close = find_closing_quote(query, i + 3, "'", backslash_escapes: @mysql)
+            if close
+              result << "?"
+              i = close + 1
+              next
+            end
+          end
+          result << char
+          i += 1
+        end
+      end
+
+      result
+    end
+
+    # Swaps every quoted identifier for a placeholder so the value
+    # replacement below cannot touch it: a column named "true" or `col2`
+    # must come back intact. A double-quoted span that is not an identifier
+    # can only remain after an unterminated literal was copied through
+    # verbatim; it is a value and is redacted here.
     def protect_identifiers(query)
       protected_identifiers = {}
       identifier_counter = 0
-      normalized = query.dup
 
-      # Protect backticked identifiers (MySQL style)
-      normalized = normalized.gsub(/`([^`]+)`/) do |match|
+      normalized = query.gsub(/`[^`]+`|"([^"]+)"/) do |match|
+        next "?" if $1 && !keep_double_quoted?($1)
+
         placeholder = "__IDENTIFIER_#{identifier_counter}__"
         protected_identifiers[placeholder] = match
         identifier_counter += 1
         placeholder
       end
 
-      # Protect double-quoted identifiers (PostgreSQL/SQL standard style)
-      # Only protect if they appear in contexts where identifiers are expected
-      normalized = normalized.gsub(/"([^"]+)"/) do |match|
-        content = $1
-        # Only protect if it looks like an identifier (no spaces, not a sentence)
-        if looks_like_identifier?(content)
-          placeholder = "__IDENTIFIER_#{identifier_counter}__"
-          protected_identifiers[placeholder] = match
-          identifier_counter += 1
-          placeholder
-        else
-          match  # Leave it as-is for now, will be replaced as string literal
-        end
-      end
-
       { normalized: normalized, mapping: protected_identifiers }
     end
 
-    def looks_like_identifier?(content)
-      content.match?(/^[a-zA-Z_][a-zA-Z0-9_]*$/) || content.include?(".")
+    def keep_double_quoted?(content)
+      !@mysql && content.length <= MAX_QUOTED_IDENTIFIER_LENGTH && content.match?(QUOTED_IDENTIFIER)
     end
 
     def replace_literal_values(query)
       normalized = query.dup
 
-      # Replace string literals (single quotes) FIRST so a very long literal
-      # collapses to a single "?" before any other regex in this method has
-      # to scan across it — see LONG_QUERY_THRESHOLD and issue #286.
-      normalized = replace_quoted_literals(normalized, "'")
-
-      # Replace double-quoted string literals (not protected identifiers)
-      normalized = replace_quoted_literals(normalized, '"')
-
-      # Replace floating-point numbers FIRST (before integers) to avoid double replacement
+      # Replace numbers in exponent form first (1e5, 2.5E-3, 1.0e-05 as Ruby
+      # writes small floats), then decimals, then integers, so that no later
+      # pattern can match the remains of an earlier one. The lookarounds keep
+      # digits that are part of an identifier (users2, int4) in place.
+      normalized = normalized.gsub(/(?<![a-zA-Z_])\b\d+(?:\.\d+)?[eE][+-]?\d+\b(?![a-zA-Z_])/, "?")
       normalized = normalized.gsub(/(?<![a-zA-Z_])\b\d+\.\d+\b(?![a-zA-Z_])/, "?")
-
-      # Replace integer literals with placeholders, but preserve identifiers containing numbers
-      # Negative lookbehind/lookahead prevents replacing numbers in table/column names
       normalized = normalized.gsub(/(?<![a-zA-Z_])\b\d+\b(?![a-zA-Z_])/, "?")
 
       # Handle boolean literals
-      normalized = normalized.gsub(/\b(true|false)\b/i, "?")
-
-      normalized
-    end
-
-    # Replaces quote_char-delimited string literals with a single placeholder.
-    # Uses a regex for the common case (fast — see benchmarks in
-    # LONG_QUERY_THRESHOLD's comment) and falls back to a manual scan for
-    # very long queries, where that regex risks Regexp::TimeoutError under
-    # an aggressively configured Regexp.timeout — see issue #286.
-    def replace_quoted_literals(query, quote_char)
-      if query.length < LONG_QUERY_THRESHOLD
-        pattern = quote_char == "'" ? /'(?:[^']|'')*'/ : /"(?:[^"]|"")*"/
-        query.gsub(pattern, "?")
-      else
-        scan_quoted_literals(query, quote_char)
-      end
-    end
-
-    # Linear, non-regex scan used above LONG_QUERY_THRESHOLD. A quote left
-    # unterminated for the rest of the string is left untouched rather than
-    # rescanned from every possible start position, which is what would make
-    # a naive manual reimplementation quadratic.
-    def scan_quoted_literals(query, quote_char)
-      result = +""
-      i = 0
-      len = query.length
-
-      while i < len
-        char = query[i]
-
-        unless char == quote_char
-          result << char
-          i += 1
-          next
-        end
-
-        close_index = find_closing_quote(query, i + 1, quote_char)
-
-        if close_index
-          result << "?"
-          i = close_index + 1
-        else
-          result << query[i..]
-          break
-        end
-      end
-
-      result
+      normalized.gsub(/\b(true|false)\b/i, "?")
     end
 
     # Returns the index of the unescaped closing quote_char starting the
     # search at `start`, treating a doubled quote_char (e.g. '') as an
-    # escaped literal quote rather than a terminator. Returns nil if the
-    # string is never closed.
-    def find_closing_quote(query, start, quote_char)
+    # escaped literal quote rather than a terminator. With
+    # backslash_escapes, a backslash also escapes the character after it.
+    # Returns nil if the string is never closed.
+    def find_closing_quote(query, start, quote_char, backslash_escapes: false)
       j = start
       len = query.length
 
       while j < len
-        if query[j] == quote_char
+        char = query[j]
+
+        if backslash_escapes && char == "\\"
+          j += 2
+        elsif char == quote_char
           return j unless query[j + 1] == quote_char
           j += 2
         else
@@ -171,42 +236,54 @@ module RailsPulse
 
     def handle_special_constructs(query)
       normalized = normalize_in_clauses(query)
+      normalized = normalize_values_rows(normalized)
       normalized.gsub(/\bBETWEEN\s+\?\s+AND\s+\?/i, "BETWEEN ? AND ?")
     end
 
-    # Replaces IN clause contents with placeholders using paren-depth tracking
-    # so that subqueries containing nested parens (e.g. HAVING (COUNT(*) > N))
-    # are handled correctly. A flat [^)]+ regex stops at the first ) it finds,
-    # which breaks when the subquery contains function calls or nested clauses.
+    # The number of values in an IN list is a property of one execution, not
+    # of the statement, so every literal list becomes IN (?): a preload that
+    # runs with 3 ids and then with 12 is one Query. A subquery is kept, with
+    # the IN clauses inside it normalized the same way, so `id IN (SELECT …)`
+    # stays a different statement from `id IN (?)`. Paren depth is tracked so
+    # a subquery containing nested parens (HAVING (COUNT(*) > ?)) ends where
+    # it should.
     def normalize_in_clauses(query)
       result = +""
       i = 0
       while i < query.length
-        at_word_boundary = i == 0 || !query[i - 1].match?(/[a-zA-Z_0-9]/i)
-        m = at_word_boundary && query[i..].match(/\AIN\s*\(\s*/i)
+        m = word_boundary?(query, i) && query[i..].match(/\AIN\s*\(/i)
 
         if m
-          content_start = i + m[0].length
-          depth = 1
-          j = content_start
-
-          while j < query.length && depth > 0
-            case query[j]
-            when "(" then depth += 1
-            when ")" then depth -= 1
-            end
-            j += 1 if depth > 0
-          end
+          open = i + m[0].length - 1
+          close = matching_paren(query, open)
+          content = query[(open + 1)...close]
+          result << (content.match?(/\bSELECT\b/i) ? "IN (#{normalize_in_clauses(content)})" : "IN (?)")
+          i = close + 1
+        else
+          result << query[i]
+          i += 1
+        end
+      end
+      result
+    end
+
+    # A multi-row insert (insert_all) lists one parenthesised group per row,
+    # and the row count is likewise a property of one execution: only the
+    # first row is kept, so VALUES (?, ?), (?, ?) becomes VALUES (?, ?).
+    def normalize_values_rows(query)
+      result = +""
+      i = 0
+      while i < query.length
+        m = word_boundary?(query, i) && query[i..].match(/\AVALUES\s*\(/i)
 
-          content = query[content_start...j]
+        if m
+          close = matching_paren(query, i + m[0].length - 1)
+          result << query[i..close]
+          i = close + 1
 
-          if content.strip.match?(/\ASELECT\s/i)
-            result << "IN (?)"
-          else
-            value_count = [ content.split(",").length, 1 ].max
-            result << "IN (#{Array.new(value_count, "?").join(", ")})"
+          while i < query.length && (row = query[i..].match(/\A\s*,\s*\(/))
+            i = matching_paren(query, i + row[0].length - 1) + 1
           end
-          i = j + 1
         else
           result << query[i]
           i += 1
@@ -215,10 +292,33 @@ module RailsPulse
       result
     end
 
+    def word_boundary?(query, i)
+      i == 0 || !query[i - 1].match?(/[a-zA-Z_0-9]/)
+    end
+
+    # Index of the ")" matching the "(" at `open`, or the query's length
+    # when it is never closed.
+    def matching_paren(query, open)
+      depth = 0
+      j = open
+      while j < query.length
+        case query[j]
+        when "(" then depth += 1
+        when ")"
+          depth -= 1
+          return j if depth == 0
+        end
+        j += 1
+      end
+      query.length
+    end
+
     def restore_identifiers(query, identifier_mapping)
       normalized = query.dup
+      # Block form: a replacement string would interpret \` or \\ inside an
+      # identifier as a back-reference.
       identifier_mapping.each do |placeholder, original|
-        normalized = normalized.gsub(placeholder, original)
+        normalized = normalized.gsub(placeholder) { original }
       end
       normalized
     end

data/app/services/rails_pulse/summary_service/from_child_periods.rb

@@ -0,0 +1,161 @@
+module RailsPulse
+  class SummaryService
+    # Summary rows for a day, week or month, rolled up from the summaries of
+    # its child periods (hours for a day, days for a week or month) without
+    # reading raw rows.
+    #
+    # The database combines the children in one GROUP BY, so Ruby receives
+    # one row per summarizable however many child periods there are. Counts,
+    # totals, averages, min, max, status counts and the standard deviation
+    # combine exactly. P50/P95/P99 are the count-weighted average of the
+    # children's, the same weighting Tables::Base applies when a dashboard
+    # range spans several periods.
+    #
+    # Only plain SUM, MIN, MAX and arithmetic are used, so the query runs
+    # unchanged on SQLite (which has no STDDEV or, without its math
+    # extension, POWER), PostgreSQL and MySQL. Divisions happen in Ruby,
+    # where integer sums cannot truncate them.
+    #
+    # Rows carry the summarizable and its metrics; SummaryService adds the
+    # period columns.
+    class FromChildPeriods
+      TABLE = "rails_pulse_summaries".freeze
+      PERCENTILES = %i[p50_duration p95_duration p99_duration].freeze
+      STATUS_COLUMNS = %i[error_count success_count status_2xx status_3xx status_4xx status_5xx].freeze
+
+      AGGREGATES = {
+        count: "SUM(#{TABLE}.count)",
+        total_duration: "SUM(#{TABLE}.total_duration)",
+        min_duration: "MIN(#{TABLE}.min_duration)",
+        max_duration: "MAX(#{TABLE}.max_duration)",
+        **PERCENTILES.flat_map { |column|
+          [
+            [ :"#{column}_weighted", "SUM(#{TABLE}.#{column} * #{TABLE}.count)" ],
+            [ :"#{column}_weight", "SUM(CASE WHEN #{TABLE}.#{column} IS NOT NULL THEN #{TABLE}.count ELSE 0 END)" ]
+          ]
+        }.to_h,
+        # The three sums Statistics.pooled_stddev needs.
+        stddev_within_sum: "SUM((#{TABLE}.count - 1) * COALESCE(#{TABLE}.stddev_duration, 0) * COALESCE(#{TABLE}.stddev_duration, 0))",
+        weighted_mean_sum: "SUM(#{TABLE}.count * #{TABLE}.avg_duration)",
+        weighted_square_sum: "SUM(#{TABLE}.count * #{TABLE}.avg_duration * #{TABLE}.avg_duration)",
+        **STATUS_COLUMNS.to_h { |column| [ column, "SUM(#{TABLE}.#{column})" ] }
+      }.freeze
+
+      # `child_starts` are the boundaries the children must start on. Rows in
+      # the range that start elsewhere were written under a different
+      # aggregation time zone; they overlap these and would be counted twice.
+      def initialize(child_period_type, child_starts)
+        @child_period_type = child_period_type
+        @child_starts = child_starts
+      end
+
+      # Always exactly one row, even for an empty period (see SummaryService).
+      def request_rows
+        combined = combined_of("RailsPulse::Request")[0]
+
+        [ row("RailsPulse::Request", 0).merge(duration_metrics(combined), status_metrics(combined)) ]
+      end
+
+      def route_rows
+        combined_of("RailsPulse::Route").map do |route_id, combined|
+          row("RailsPulse::Route", route_id).merge(duration_metrics(combined), status_metrics(combined))
+        end
+      end
+
+      def query_rows
+        combined_of("RailsPulse::Query").map do |query_id, combined|
+          row("RailsPulse::Query", query_id).merge(duration_metrics(combined))
+        end
+      end
+
+      def job_rows
+        combined_by_job = combined_of("RailsPulse::Job")
+        return [] if combined_by_job.empty?
+
+        # A job whose row has gone (count-based cleanup) has nothing to
+        # summarize against.
+        known_job_ids = Job.where(id: combined_by_job.keys).pluck(:id).to_set
+
+        combined_by_job.filter_map do |job_id, combined|
+          next unless known_job_ids.include?(job_id)
+
+          row("RailsPulse::Job", job_id).merge(
+            duration_metrics(combined),
+            status_metrics(combined).slice(:error_count, :success_count)
+          )
+        end
+      end
+
+      # The children already carry the all-groups row (id 0), which sums like
+      # any other group.
+      def exception_rows
+        combined_of("RailsPulse::ExceptionGroup").map do |group_id, combined|
+          row("RailsPulse::ExceptionGroup", group_id).merge(count: combined[:count].to_i)
+        end
+      end
+
+      private
+
+      def row(summarizable_type, summarizable_id)
+        { summarizable_type: summarizable_type, summarizable_id: summarizable_id }
+      end
+
+      # { summarizable_type => { summarizable_id => { aggregate => value } } }.
+      # Empty children (an idle hour's count-0 heartbeat) are left out; they
+      # add nothing, and an idle period falls back to empty metrics.
+      def combined_children
+        @combined_children ||= Summary
+          .where(period_type: @child_period_type, period_start: @child_starts)
+          .where(count: 1..)
+          .group(:summarizable_type, :summarizable_id)
+          .pluck(:summarizable_type, :summarizable_id, *AGGREGATES.values.map { |sql| Arel.sql(sql) })
+          .each_with_object({}) do |(type, id, *values), by_type|
+            (by_type[type] ||= {})[id] = AGGREGATES.keys.zip(values).to_h
+          end
+      end
+
+      # { summarizable_id => combined } for one summarizable type.
+      def combined_of(summarizable_type)
+        combined_children.fetch(summarizable_type, {})
+      end
+
+      def duration_metrics(combined)
+        return Metrics.duration([]) unless combined
+
+        count = combined[:count].to_i
+        total = combined[:total_duration].to_f
+
+        {
+          count: count,
+          # Count-weighted mean of the children's averages: identical to
+          # total / count where every row has a duration (requests, routes,
+          # queries), and consistent with the children for jobs, whose count
+          # includes runs with no recorded duration (discarded runs) that
+          # total / count would dilute the average with.
+          avg_duration: combined[:weighted_mean_sum].to_f / count,
+          min_duration: combined[:min_duration]&.to_f,
+          max_duration: combined[:max_duration]&.to_f,
+          total_duration: total,
+          **PERCENTILES.to_h { |column| [ column, weighted_percentile(combined, column) ] },
+          stddev_duration: Statistics.pooled_stddev(
+            count: count,
+            within_sum: combined[:stddev_within_sum],
+            weighted_mean_sum: combined[:weighted_mean_sum],
+            weighted_square_sum: combined[:weighted_square_sum]
+          )
+        }
+      end
+
+      def weighted_percentile(combined, column)
+        weight = combined[:"#{column}_weight"].to_f
+        return nil if weight.zero?
+
+        combined[:"#{column}_weighted"].to_f / weight
+      end
+
+      def status_metrics(combined)
+        STATUS_COLUMNS.index_with { |column| combined ? combined[column].to_i : 0 }
+      end
+    end
+  end
+end