chronos-ruby 0.9.0.pre.4 → 1.1.0

data/lib/chronos/application/apm_aggregator.rb

@@ -13,28 +13,51 @@ module Chronos
     #   batches = aggregator.flush
     # @errors Invalid observations are ignored and never escape to the application.
     # @performance Group, transaction, query, bucket, and batch counts are strictly bounded.
-    class ApmAggregator
+    class ApmAggregator # rubocop:disable Metrics/ClassLength
       include ApmErrorClassifier
 
       METRIC_TYPES = %w(request query job external_http).freeze
-      def initialize(config)
+      QUERY_ERROR_SIGNALS = {
+        "connection_error" => /Connection|NoDatabase|Adapter/i,
+        "deadlock" => /Deadlock/i,
+        "query_timeout" => /StatementTimeout|QueryCanceled|QueryTimeout|Timeout/i,
+        "pool_timeout" => /ConnectionTimeout|PoolTimeout/i,
+        "lock_timeout" => /LockWaitTimeout|LockTimeout/i,
+        "constraint_violation" => /RecordNotUnique|NotNullViolation|ForeignKeyViolation|InvalidForeignKey|Constraint/i
+      }.freeze
+      SIGNAL_DIAGNOSTICS = {
+        "slow_query" => ["warning", "performance", "Query duration reached the slow threshold"].freeze,
+        "long_transaction" => ["warning", "transaction", "Transaction SQL reached the long threshold"].freeze,
+        "connection_error" => ["error", "connection", "Database connection failed"].freeze,
+        "deadlock" => ["error", "locking", "The database reported a deadlock"].freeze,
+        "query_timeout" => ["error", "timeout", "The database query timed out"].freeze,
+        "pool_timeout" => ["error", "connection_pool", "Database connection-pool checkout timed out"].freeze,
+        "lock_timeout" => ["error", "locking", "The database lock wait timed out"].freeze,
+        "constraint_violation" => ["error", "constraint", "The database rejected a constraint"].freeze
+      }.freeze
+      def initialize(config, options = {})
         @config = config
+        @clock = options[:clock] || proc { Time.now.to_f }
         @mutex = Mutex.new
         @groups = {}
         @transactions = {}
         @observations = 0
         @dropped_groups = 0
+        @dropped_trace_trackers = 0
+        @expired_trace_trackers = 0
+        @dropped_query_fingerprints = 0
       end
 
       def record(event_type, payload = {}, context = {})
         return [] unless @config.apm_enabled
 
         @mutex.synchronize do
+          expire_transactions
           type = event_type.to_s
           data = hash(payload)
           execution = hash(context)
-          observe_component(type, data, execution)
-          add_metric(type, data, execution) if aggregate_metric?(type, data)
+          correlation = observe_component(type, data, execution)
+          add_metric(type, data, execution, correlation) if aggregate_metric?(type, data)
           @observations += 1 if aggregate_metric?(type, data)
           @observations >= @config.apm_flush_count ? drain_locked : []
         end
@@ -53,6 +76,9 @@ module Chronos
           {
             "groups" => @groups.length, "dropped_groups" => @dropped_groups,
             "transactions" => @transactions.length,
+            "dropped_trace_trackers" => @dropped_trace_trackers,
+            "expired_trace_trackers" => @expired_trace_trackers,
+            "dropped_query_fingerprints" => @dropped_query_fingerprints,
             "tracked_queries" => @transactions.values.inject(0) do |total, transaction|
               total + transaction["queries"].length
             end
@@ -66,7 +92,7 @@ module Chronos
         METRIC_TYPES.include?(type) && !(type == "request" && payload["kind"].to_s == "view")
       end
 
-      def add_metric(type, payload, context)
+      def add_metric(type, payload, context, correlation)
         dimensions = dimensions_for(type, payload)
         key = metric_key(type, dimensions)
         aggregate = @groups[key]
@@ -80,53 +106,77 @@ module Chronos
         end
         duration = non_negative(payload["duration_ms"] || payload[:duration_ms])
         status = ["request", "external_http"].include?(type) ? payload["status"] || payload[:status] : nil
-        signals = signals_for(type, payload, context, duration)
+        signals = signals_for(type, payload, context, duration, correlation)
+        diagnostics = diagnostics_for(type, payload, context, signals, correlation)
         breakdown = breakdown_for(type, payload, context, duration)
         aggregate.observe(
-          duration, error?(type, payload), breakdown, signals, status
+          duration, error?(type, payload), breakdown, signals,
+          :status => status, :diagnostics => diagnostics,
+          :query_analysis => type == "query" ? hash(payload["analysis"] || payload[:analysis]) : {}
         )
       end
 
       def observe_component(type, payload, context)
-        return if type == "job"
+        return {} if type == "job"
 
         trace_id = trace_id(context)
-        return if trace_id.empty?
+        return {} if trace_id.empty?
 
         transaction = transaction_for(trace_id)
-        return unless transaction
+        return {} unless transaction
+
+        transaction["last_seen_at"] = now
 
         category = component_category(type, payload)
         return observe_query(transaction, payload) if type == "query" && category.nil?
-        return unless category
+        return {} unless category
 
         transaction["breakdown_ms"][category] ||= 0.0
         transaction["breakdown_ms"][category] += non_negative(payload["duration_ms"] || payload[:duration_ms])
-        observe_query(transaction, payload) if type == "query"
+        return observe_query(transaction, payload) if type == "query"
+
+        {}
       end
 
       def transaction_for(trace_id)
         existing = @transactions[trace_id]
         return existing if existing
-        return nil if @transactions.length >= @config.apm_max_groups
+        if @transactions.length >= @config.apm_max_groups
+          @dropped_trace_trackers += 1
+          return nil
+        end
 
-        @transactions[trace_id] = {"breakdown_ms" => {}, "signals" => {}, "queries" => {}}
+        @transactions[trace_id] = {
+          "breakdown_ms" => {}, "signals" => {}, "queries" => {}, "diagnostics" => [],
+          "last_seen_at" => now
+        }
       end
 
       def observe_query(transaction, payload)
         fingerprint = (payload["fingerprint"] || payload[:fingerprint]).to_s
-        return if fingerprint.empty?
+        return {} if fingerprint.empty?
 
         queries = transaction["queries"]
-        return unless queries.key?(fingerprint) || queries.length < @config.apm_max_queries_per_request
+        unless queries.key?(fingerprint) || queries.length < @config.apm_max_queries_per_request
+          @dropped_query_fingerprints += 1
+          return {}
+        end
 
         queries[fingerprint] ||= 0
         queries[fingerprint] += 1
         count = queries[fingerprint]
         slow = non_negative(payload["duration_ms"]) >= @config.apm_slow_query_threshold_ms
         increment_signal(transaction, "slow_query") if slow
-        increment_signal(transaction, "repeated_query") if count > 1
-        increment_signal(transaction, "possible_n_plus_one") if count == @config.apm_n_plus_one_threshold
+        current = {}
+        if count > 1
+          increment_signal(transaction, "repeated_query")
+          current["repeated_query"] = 1
+        end
+        if n_plus_one_candidate?(payload) && count == @config.apm_n_plus_one_threshold
+          increment_signal(transaction, "possible_n_plus_one")
+          current["possible_n_plus_one"] = 1
+        end
+        {"signals" => current, "query_count" => count}
       end
 
       def breakdown_for(type, payload, context, duration)
@@ -145,13 +195,75 @@ module Chronos
         explicit
       end
 
-      def signals_for(type, payload, context, duration)
+      def signals_for(type, payload, context, duration, correlation)
         return request_signals(context) if type == "request"
-        return query_signals(payload, duration) if type == "query"
+        return merge_integer_signals(query_signals(payload, duration), hash(correlation["signals"])) if type == "query"
 
         {}
       end
 
+      def diagnostics_for(type, payload, context, signals, correlation)
+        return request_diagnostics(context) if type == "request"
+        return [] unless type == "query"
+
+        analysis = hash(payload["analysis"] || payload[:analysis])
+        result = Array(analysis["diagnostics"] || analysis[:diagnostics]).first(20)
+        result += signal_diagnostics(signals, correlation)
+        error_class = (payload["error_class"] || payload[:error_class]).to_s
+        unless error_class.empty?
+          result << diagnostic("query_execution_error", "error", "execution",
+                               "The database query raised an exception", "error_class" => error_class[0, 128])
+        end
+        retain_transaction_diagnostics(context, result)
+        result.first(20)
+      end
+
+      def signal_diagnostics(signals, correlation)
+        result = signal_error_diagnostics(signals)
+        if hash(correlation["signals"])["repeated_query"]
+          result << diagnostic("repeated_query", "info", "query_pattern",
+                               "The same query fingerprint repeated in one trace",
+                               "occurrence" => correlation["query_count"])
+        end
+        if hash(correlation["signals"])["possible_n_plus_one"]
+          result << diagnostic("possible_n_plus_one", "warning", "n_plus_one",
+                               "Repeated SELECT reached the N+1 threshold",
+                               "occurrence" => correlation["query_count"])
+          item = diagnostic("n_plus_one_eager_loading", "suggestion", "n_plus_one",
+                            "Review eager loading or batch loading for this query fingerprint")
+          item["recommendation"] = "Use includes, preload, eager_load, or an equivalent bounded batch query " \
+                                   "when semantics allow"
+          result << item
+        end
+        result
+      end
+
+      def signal_error_diagnostics(signals)
+        SIGNAL_DIAGNOSTICS.each_with_object([]) do |(code, values), result|
+          result << diagnostic(code, values[0], values[1], values[2]) if signals[code]
+        end
+      end
+
+      def diagnostic(code, severity, category, message, evidence = {})
+        {"code" => code, "severity" => severity, "category" => category,
+         "message" => message, "evidence" => evidence}
+      end
+
+      def retain_transaction_diagnostics(context, diagnostics)
+        transaction = @transactions[trace_id(context)]
+        return unless transaction
+
+        diagnostics.each do |item|
+          next if transaction["diagnostics"].any? { |existing| existing["code"] == item["code"] }
+          transaction["diagnostics"] << item if transaction["diagnostics"].length < 20
+        end
+      end
+
+      def request_diagnostics(context)
+        transaction = @transactions[trace_id(context)]
+        transaction ? transaction["diagnostics"].dup : []
+      end
+
       def request_signals(context)
         transaction = @transactions[trace_id(context)]
         transaction ? transaction["signals"].dup : {}
@@ -162,16 +274,30 @@ module Chronos
         signals["slow_query"] = 1 if duration >= @config.apm_slow_query_threshold_ms
         signals["long_transaction"] = 1 if long_transaction?(payload, duration)
         error_class = (payload["error_class"] || payload[:error_class]).to_s
-        signals["connection_error"] = 1 if error_class =~ /Connection|NoDatabase|Adapter/i
-        signals["deadlock"] = 1 if error_class =~ /Deadlock/i
+        QUERY_ERROR_SIGNALS.each do |name, pattern|
+          signals[name] = 1 if error_class =~ pattern
+        end
         signals
       end
 
       def long_transaction?(payload, duration)
         operation = (payload["operation"] || payload[:operation]).to_s
         name = (payload["name"] || payload[:name]).to_s
-        transaction = operation == "BEGIN" || operation == "COMMIT" || name =~ /TRANSACTION/i
-        transaction && duration >= @config.apm_long_transaction_threshold_ms
+        transaction = operation == "BEGIN" || operation == "COMMIT" || operation == "ROLLBACK" || name =~ /TRANSACTION/i
+        measured = non_negative(payload["transaction_duration_ms"] || payload[:transaction_duration_ms] || duration)
+        transaction && measured >= @config.apm_long_transaction_threshold_ms
+      end
+
+      def n_plus_one_candidate?(payload)
+        operation = (payload["operation"] || payload[:operation]).to_s
+        cached = payload.key?("cached") ? payload["cached"] : payload[:cached]
+        operation == "SELECT" && cached != true
+      end
+
+      def merge_integer_signals(left, right)
+        result = left.dup
+        right.each { |key, value| result[key.to_s] = result.fetch(key.to_s, 0) + value.to_i }
+        result
       end
 
       def dimensions_for(type, payload)
@@ -238,23 +364,47 @@ module Chronos
 
       def drain_locked
         metrics = @groups.values.map(&:to_h)
-        if metrics.empty?
-          @transactions = {}
-          return []
-        end
+        return [] if metrics.empty?
 
         dropped = @dropped_groups
         @groups = {}
-        @transactions = {}
         @observations = 0
         @dropped_groups = 0
+        tracking = {
+          "active_traces" => @transactions.length,
+          "dropped_trace_trackers" => @dropped_trace_trackers,
+          "expired_trace_trackers" => @expired_trace_trackers,
+          "dropped_query_fingerprints" => @dropped_query_fingerprints
+        }
+        @dropped_trace_trackers = 0
+        @expired_trace_trackers = 0
+        @dropped_query_fingerprints = 0
         batches = []
         metrics.each_slice(@config.apm_batch_size) do |slice|
+          slice.first["tracking"] = batches.empty? ? tracking : empty_tracking
           batches << {"metrics" => slice, "dropped_groups" => batches.empty? ? dropped : 0}
         end
         batches
       end
 
+      def expire_transactions
+        threshold = now - @config.apm_trace_ttl_seconds.to_f
+        expired = @transactions.keys.select { |key| @transactions[key]["last_seen_at"].to_f < threshold }
+        expired.each { |key| @transactions.delete(key) }
+        @expired_trace_trackers += expired.length
+      end
+
+      def empty_tracking
+        {"active_traces" => @transactions.length, "dropped_trace_trackers" => 0,
+         "expired_trace_trackers" => 0, "dropped_query_fingerprints" => 0}
+      end
+
+      def now
+        @clock.call.to_f
+      rescue StandardError
+        Time.now.to_f
+      end
+
       def non_negative(value)
         number = value.to_f
         number < 0.0 ? 0.0 : number

data/lib/chronos/configuration/apm_validation.rb

@@ -9,7 +9,7 @@ module Chronos
     # @thread_safety Reads one mutable configuration instance without shared state.
     # @compatibility Ruby 2.2.10 through Ruby 2.6.
     # @errors Invalid values become messages and never raise from validation.
-    module ApmConfigurationValidation
+    module ApmConfigurationValidation # rubocop:disable Metrics/ModuleLength
       private
 
       def apm_errors
@@ -32,6 +32,24 @@ module Chronos
         unless positive_integer?(apm_max_queries_per_request)
           errors << "apm_max_queries_per_request must be a positive integer"
         end
+        errors.concat(query_capacity_errors)
+        errors
+      end
+
+      def query_capacity_errors
+        errors = []
+        unless apm_query_analysis_max_queries.is_a?(Integer) &&
+               apm_query_analysis_max_queries >= 1 && apm_query_analysis_max_queries <= 500
+          errors << "apm_query_analysis_max_queries must be between 1 and 500"
+        end
+        unless apm_query_inspection_max_queries.is_a?(Integer) &&
+               apm_query_inspection_max_queries >= 1 && apm_query_inspection_max_queries <= 100
+          errors << "apm_query_inspection_max_queries must be between 1 and 100"
+        end
+        unless apm_transaction_max_connections.is_a?(Integer) &&
+               apm_transaction_max_connections >= 1 && apm_transaction_max_connections <= 500
+          errors << "apm_transaction_max_connections must be between 1 and 500"
+        end
         errors
       end
 
@@ -46,9 +64,41 @@ module Chronos
         unless apm_n_plus_one_threshold.is_a?(Integer) && apm_n_plus_one_threshold >= 2
           errors << "apm_n_plus_one_threshold must be an integer greater than or equal to 2"
         end
+        errors.concat(query_threshold_errors)
         errors
       end
 
+      def query_threshold_errors
+        errors = []
+        unless positive_number?(apm_trace_ttl_seconds)
+          errors << "apm_trace_ttl_seconds must be greater than zero"
+        end
+        unless non_negative_number?(apm_query_inspection_min_duration_ms)
+          errors << "apm_query_inspection_min_duration_ms must be zero or greater"
+        end
+        query_analysis_boolean_attributes.each do |name, value|
+          errors << "#{name} must be true or false" unless boolean?(value)
+        end
+        if (apm_query_statistics_enabled || apm_query_plan_enabled) && !apm_query_inspection_enabled
+          errors << "query statistics and plans require apm_query_inspection_enabled"
+        end
+        errors
+      end
+
+      def query_analysis_boolean_attributes
+        {
+          :apm_query_analysis_enabled => apm_query_analysis_enabled,
+          :apm_query_inspection_enabled => apm_query_inspection_enabled,
+          :apm_query_statistics_enabled => apm_query_statistics_enabled,
+          :apm_query_plan_enabled => apm_query_plan_enabled,
+          :apm_transaction_tracking_enabled => apm_transaction_tracking_enabled
+        }
+      end
+
+      def non_negative_number?(value)
+        value.is_a?(Numeric) && value >= 0
+      end
+
       def increasing_positive_numbers?(values)
         return false unless values.is_a?(Array) && !values.empty? && values.length <= 19
         return false unless values.all? { |value| positive_number?(value) }

data/lib/chronos/configuration.rb

@@ -47,7 +47,13 @@ module Chronos
       :apm_enabled, :apm_max_groups, :apm_flush_count, :apm_batch_size,
       :apm_max_queries_per_request, :apm_slow_query_threshold_ms,
       :apm_long_transaction_threshold_ms, :apm_n_plus_one_threshold,
-      :apm_histogram_buckets, :external_http_enabled, :external_http_trace_headers,
+      :apm_histogram_buckets, :apm_trace_ttl_seconds,
+      :apm_query_analysis_enabled, :apm_query_inspection_enabled,
+      :apm_query_analysis_max_queries,
+      :apm_query_statistics_enabled, :apm_query_plan_enabled,
+      :apm_query_inspection_min_duration_ms, :apm_query_inspection_max_queries,
+      :apm_transaction_tracking_enabled, :apm_transaction_max_connections,
+      :external_http_enabled, :external_http_trace_headers,
       :cache_key_mode, :dependency_reporting, :dependency_max_items
     ].freeze
 
@@ -146,6 +152,16 @@ module Chronos
       @apm_long_transaction_threshold_ms = 1000.0
       @apm_n_plus_one_threshold = 5
       @apm_histogram_buckets = [5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0]
+      @apm_trace_ttl_seconds = 60.0
+      @apm_query_analysis_enabled = true
+      @apm_query_analysis_max_queries = 100
+      @apm_query_inspection_enabled = false
+      @apm_query_statistics_enabled = false
+      @apm_query_plan_enabled = false
+      @apm_query_inspection_min_duration_ms = 500.0
+      @apm_query_inspection_max_queries = 20
+      @apm_transaction_tracking_enabled = true
+      @apm_transaction_max_connections = 100
     end
 
     def initialize_observability_defaults

data/lib/chronos/core/metric_aggregate.rb

@@ -2,18 +2,20 @@ module Chronos
   module Core
     # Accumulates one bounded APM metric group and serializes aggregate statistics.
     #
-    # @responsibility Track counts, errors, durations, histogram, breakdown, and signals.
+    # @responsibility Track counts, errors, durations, histogram, breakdown, signals, and diagnostics.
     # @motivation Keep numerical accumulation separate from grouping and request correlation.
-    # @limits It does not choose dimensions, retain observations, or calculate percentiles.
+    # @limits It does not choose dimensions, retain observations, or calculate exact percentiles.
     # @collaborators ApmAggregator and immutable histogram boundaries.
     # @thread_safety Mutable by design; callers must synchronize access.
     # @compatibility Ruby 2.2.10 through Ruby 2.6.
     # @example
-    #   metric.observe(12.0, false, {"database" => 3.0}, {})
+    #   metric.observe(12.0, false, {"database" => 3.0}, {}, :diagnostics => [])
     # @errors Non-numeric durations become zero and never escape.
     # @performance Memory is fixed by the configured histogram boundary count.
     class MetricAggregate
       BREAKDOWN_CATEGORIES = %w(database view external_http cache queue application unknown).freeze
+      DIAGNOSTIC_SEVERITIES = %w(error warning info suggestion).freeze
+      MAX_DIAGNOSTICS = 20
 
       def initialize(metric_type, dimensions, boundaries)
         @metric_type = metric_type
@@ -28,9 +30,12 @@ module Chronos
         @breakdown = {}
         @signals = {}
         @status_codes = {}
+        @diagnostics = {}
+        @severity_counts = {}
+        @query_analysis = nil
       end
 
-      def observe(duration, error, breakdown, signals, status = nil)
+      def observe(duration, error, breakdown, signals, options = {})
         value = non_negative(duration)
         @count += 1
         @error_count += 1 if error
@@ -41,7 +46,9 @@ module Chronos
         @buckets[bucket || @boundaries.length] += 1
         add_breakdown(breakdown)
         add_signals(signals)
-        add_status(status)
+        add_status(options[:status])
+        add_diagnostics(options[:diagnostics])
+        retain_query_analysis(options[:query_analysis])
         self
       end
 
@@ -50,9 +57,10 @@ module Chronos
           "metric_type" => @metric_type, "dimensions" => @dimensions,
           "count" => @count, "error_count" => @error_count,
           "error_rate" => (@error_count.to_f / @count).round(6),
-          "duration_ms" => duration_summary, "histogram" => histogram,
+          "duration_ms" => duration_summary, "percentiles_ms" => percentiles, "histogram" => histogram,
           "breakdown_ms" => rounded_hash(@breakdown), "signals" => @signals,
-          "status_codes" => @status_codes
+          "status_codes" => @status_codes, "severity_counts" => @severity_counts,
+          "diagnostics" => @diagnostics.values, "query_analysis" => @query_analysis || {}
         }
       end
 
@@ -81,6 +89,46 @@ module Chronos
         @status_codes[key] += 1
       end
 
+      def add_diagnostics(values)
+        Array(values).first(MAX_DIAGNOSTICS).each do |diagnostic|
+          data = hash(diagnostic)
+          severity = data["severity"].to_s
+          next unless DIAGNOSTIC_SEVERITIES.include?(severity)
+
+          @severity_counts[severity] ||= 0
+          @severity_counts[severity] += 1
+          key = diagnostic_key(data)
+          existing = @diagnostics[key]
+          if existing
+            existing["count"] += 1
+          elsif @diagnostics.length < MAX_DIAGNOSTICS
+            @diagnostics[key] = data.merge("count" => 1)
+          end
+        end
+      end
+
+      def diagnostic_key(data)
+        evidence = hash(data["evidence"])
+        [data["code"], data["severity"], evidence["table"], Array(evidence["columns"]).join(",")].join("|")
+      end
+
+      def retain_query_analysis(value)
+        return unless value.is_a?(Hash) && !value.empty?
+        return if @query_analysis && analysis_score(@query_analysis) >= analysis_score(value)
+
+        @query_analysis = value.reject { |key, _child| key.to_s == "diagnostics" }
+      end
+
+      def analysis_score(value)
+        data = hash(value)
+        inspection = hash(data["inspection"] || data[:inspection])
+        score = inspection.empty? ? 0 : 1
+        score += 1 unless Array(inspection["indexes"] || inspection[:indexes]).empty?
+        score += 1 unless hash(inspection["statistics"] || inspection[:statistics]).empty?
+        score += 1 unless hash(inspection["plan"] || inspection[:plan]).empty?
+        score
+      end
+
       def duration_summary
         {
           "total" => @total.round(3), "min" => @min.round(3), "max" => @max.round(3),
@@ -94,6 +142,20 @@ module Chronos
         end
       end
 
+      def percentiles
+        {"p50" => percentile(0.50), "p95" => percentile(0.95), "p99" => percentile(0.99)}
+      end
+
+      def percentile(ratio)
+        target = (@count * ratio).ceil
+        accumulated = 0
+        @buckets.each_with_index do |count, index|
+          accumulated += count
+          return (@boundaries[index] || @max).to_f.round(3) if accumulated >= target
+        end
+        @max.to_f.round(3)
+      end
+
       def rounded_hash(values)
         values.each_with_object({}) { |(key, value), result| result[key] = value.round(3) }
       end