chronos-ruby 0.5.0.pre.1 → 0.7.0.pre.1

data/docs/troubleshooting.md

@@ -24,6 +24,26 @@ Require `chronos/rails` from the generated initializer and confirm it runs befor
 
 This is intentional. Version 0.5 records SQL operation name/duration and cache operation/store/hit status without raw statements, binds, keys, or values. These omissions are privacy and cardinality boundaries, not capture failures.
 
+## Sidekiq telemetry is missing
+
+Require `chronos/sidekiq` after Sidekiq is available and configure Chronos before jobs run. The entry point uses Sidekiq's public client/server middleware configuration and remains optional. Requiring only `chronos` or `chronos/rails` does not load Sidekiq. A job already enqueued before client middleware installation may lack propagated trace context, but server timing and failure capture can still run.
+
+## A failed Sidekiq job appears twice
+
+The bundled middleware does not install a global error handler and uses `notify_once` inside a shared job scope. Check for application-installed Chronos calls or third-party global handlers outside that scope. Keep a single bundled server middleware entry in the Sidekiq chain.
+
+## Request, query, or job events are not sent immediately
+
+This is expected with version 0.7 APM aggregation. Groups drain after `apm_flush_count` observations or during `Chronos.flush`/`Chronos.close`. Inspect `agent.diagnostics[:apm]` when using an explicit agent. Set `apm_enabled = false` only for temporary individual-event diagnostics.
+
+## Query metrics contain no literal values or binds
+
+This is intentional. The normalizer removes common literal forms and never reads binds. Queries differing only by values should share a fingerprint. If an unsupported database dialect leaves a sensitive literal form, stop delivery, add a synthetic privacy fixture, and report the dialect through the security/support process.
+
+## Possible N+1 or deadlock signal is inaccurate
+
+Local detectors emit bounded heuristics, not confirmed diagnoses. Repetition can be legitimate and exception class names can be adapter-specific. Confirm the trace in the SaaS and application logs before changing application behavior.
+
 ## Context appears missing
 
 The legacy context store is thread-local. A new application-created thread does not inherit context. Establish a new `Chronos.with_context` scope inside that thread, and issue manual notification before the scope exits. For Rack capture, supply user and explicit parameters through the documented environment keys.

data/lib/chronos/agent.rb

@@ -67,6 +67,41 @@ module Chronos
       @telemetry.call(event_type, payload, telemetry_context(context))
     end
 
+    def record_event_once(key, event_type, payload = {}, context = {})
+      execution = @context_store.get
+      captured = execution[:__chronos_captured_events] || {}
+      return false if captured[key.to_s]
+
+      captured[key.to_s] = true
+      @context_store.set(execution.merge(:__chronos_captured_events => captured))
+      record_event(event_type, payload, context)
+    rescue StandardError
+      false
+    end
+
+    def apm_integration_options
+      {
+        :enabled => @config.apm_enabled,
+        :slow_query_threshold_ms => @config.apm_slow_query_threshold_ms,
+        :root_directory => @config.root_directory
+      }
+    end
+
+    # Returns the correlation subset safe for an integration-owned process boundary.
+    def propagation_context
+      current = context_hash(@context_store.get)
+      nested = context_hash(current[:context] || current["context"])
+      request = context_hash(nested["request"] || nested[:request])
+      values = {
+        "trace_id" => nested["trace_id"] || nested[:trace_id],
+        "request_id" => nested["request_id"] || nested[:request_id] ||
+                        request["request_id"] || request[:request_id]
+      }
+      values.delete_if { |_key, value| value.to_s.empty? }
+    rescue StandardError
+      {}
+    end
+
     def notify_once(exception, context = {})
       execution = @context_store.get
       captured = execution[:__chronos_captured_exceptions] || {}
@@ -89,6 +124,7 @@ module Chronos
     end
 
     def flush(timeout = DEFAULT_FLUSH_TIMEOUT)
+      @telemetry.flush
       @delivery_pipeline.flush(timeout)
     rescue StandardError => error
       @logger.warn("Chronos flush failed: #{error.class}")
@@ -96,6 +132,7 @@ module Chronos
     end
 
     def close(timeout = DEFAULT_FLUSH_TIMEOUT)
+      @telemetry.flush
       @delivery_pipeline.close(timeout)
     rescue StandardError => error
       @logger.warn("Chronos close failed: #{error.class}")
@@ -104,6 +141,7 @@ module Chronos
 
     def diagnostics
       details = @delivery_pipeline.diagnostics
+      details[:apm] = @telemetry.diagnostics
       details[:queue].merge(details)
     end
 
@@ -124,6 +162,8 @@ module Chronos
       merged = deep_merge(context_hash(@context_store.get), context_hash(additional))
       merged.delete(:__chronos_captured_exceptions)
       merged.delete("__chronos_captured_exceptions")
+      merged.delete(:__chronos_captured_events)
+      merged.delete("__chronos_captured_events")
       buffer = merged.delete(:__chronos_breadcrumbs) || merged.delete("__chronos_breadcrumbs")
       if buffer.respond_to?(:to_a)
         merged[:context] = context_hash(merged[:context]).merge("breadcrumbs" => buffer.to_a)

data/lib/chronos/application/apm_aggregator.rb

@@ -0,0 +1,273 @@
+module Chronos
+  module Application
+    # Aggregates bounded request, query, and job observations into metric batches.
+    #
+    # @responsibility Maintain statistics, histograms, breakdown, and local query signals.
+    # @motivation Avoid one network event per observation while retaining diagnostic value.
+    # @limits Percentiles and heavy correlation remain server-side; state is process-local.
+    # @collaborators CaptureTelemetry and immutable Chronos configuration.
+    # @thread_safety A mutex protects groups, request trackers, counters, and drains.
+    # @compatibility Ruby 2.2.10 through Ruby 2.6; independent of Rails.
+    # @example
+    #   aggregator.record("request", payload, context)
+    #   batches = aggregator.flush
+    # @errors Invalid observations are ignored and never escape to the application.
+    # @performance Group, transaction, query, bucket, and batch counts are strictly bounded.
+    class ApmAggregator
+      METRIC_TYPES = %w(request query job).freeze
+      def initialize(config)
+        @config = config
+        @mutex = Mutex.new
+        @groups = {}
+        @transactions = {}
+        @observations = 0
+        @dropped_groups = 0
+      end
+
+      def record(event_type, payload = {}, context = {})
+        return [] unless @config.apm_enabled
+
+        @mutex.synchronize do
+          type = event_type.to_s
+          data = hash(payload)
+          execution = hash(context)
+          observe_component(type, data, execution)
+          add_metric(type, data, execution) if aggregate_metric?(type, data)
+          @observations += 1 if aggregate_metric?(type, data)
+          @observations >= @config.apm_flush_count ? drain_locked : []
+        end
+      rescue StandardError
+        []
+      end
+
+      def flush
+        @mutex.synchronize { drain_locked }
+      rescue StandardError
+        []
+      end
+
+      def diagnostics
+        @mutex.synchronize do
+          {
+            "groups" => @groups.length, "dropped_groups" => @dropped_groups,
+            "transactions" => @transactions.length,
+            "tracked_queries" => @transactions.values.inject(0) do |total, transaction|
+              total + transaction["queries"].length
+            end
+          }
+        end
+      end
+
+      private
+
+      def aggregate_metric?(type, payload)
+        METRIC_TYPES.include?(type) && !(type == "request" && payload["kind"].to_s == "view")
+      end
+
+      def add_metric(type, payload, context)
+        dimensions = dimensions_for(type, payload)
+        key = metric_key(type, dimensions)
+        aggregate = @groups[key]
+        unless aggregate
+          if @groups.length >= @config.apm_max_groups
+            @dropped_groups += 1
+            return
+          end
+          aggregate = Core::MetricAggregate.new(type, dimensions, @config.apm_histogram_buckets)
+          @groups[key] = aggregate
+        end
+        duration = non_negative(payload["duration_ms"] || payload[:duration_ms])
+        status = type == "request" ? payload["status"] || payload[:status] : nil
+        signals = signals_for(type, payload, context, duration)
+        breakdown = breakdown_for(type, payload, context, duration)
+        aggregate.observe(
+          duration, error?(type, payload), breakdown, signals, status
+        )
+      end
+
+      def observe_component(type, payload, context)
+        return if type == "job"
+
+        trace_id = trace_id(context)
+        return if trace_id.empty?
+
+        transaction = transaction_for(trace_id)
+        return unless transaction
+
+        category = component_category(type, payload)
+        return observe_query(transaction, payload) if type == "query" && category.nil?
+        return unless category
+
+        transaction["breakdown_ms"][category] ||= 0.0
+        transaction["breakdown_ms"][category] += non_negative(payload["duration_ms"] || payload[:duration_ms])
+        observe_query(transaction, payload) if type == "query"
+      end
+
+      def transaction_for(trace_id)
+        existing = @transactions[trace_id]
+        return existing if existing
+        return nil if @transactions.length >= @config.apm_max_groups
+
+        @transactions[trace_id] = {"breakdown_ms" => {}, "signals" => {}, "queries" => {}}
+      end
+
+      def observe_query(transaction, payload)
+        fingerprint = (payload["fingerprint"] || payload[:fingerprint]).to_s
+        return if fingerprint.empty?
+
+        queries = transaction["queries"]
+        return unless queries.key?(fingerprint) || queries.length < @config.apm_max_queries_per_request
+
+        queries[fingerprint] ||= 0
+        queries[fingerprint] += 1
+        count = queries[fingerprint]
+        slow = non_negative(payload["duration_ms"]) >= @config.apm_slow_query_threshold_ms
+        increment_signal(transaction, "slow_query") if slow
+        increment_signal(transaction, "repeated_query") if count > 1
+        increment_signal(transaction, "possible_n_plus_one") if count == @config.apm_n_plus_one_threshold
+      end
+
+      def breakdown_for(type, payload, context, duration)
+        explicit = hash(payload["breakdown_ms"] || payload[:breakdown_ms]).dup
+        if type == "request"
+          transaction = @transactions.delete(trace_id(context))
+          explicit = merge_numeric(explicit, transaction && transaction["breakdown_ms"])
+          accounted = explicit.values.inject(0.0) { |total, value| total + non_negative(value) }
+          explicit["application"] = [duration - accounted, 0.0].max
+        elsif type == "query"
+          explicit["database"] = duration
+        elsif type == "job"
+          explicit["queue"] = non_negative(payload["queue_latency_ms"] || payload[:queue_latency_ms])
+          explicit["application"] = duration
+        end
+        explicit
+      end
+
+      def signals_for(type, payload, context, duration)
+        return request_signals(context) if type == "request"
+        return query_signals(payload, duration) if type == "query"
+
+        {}
+      end
+
+      def request_signals(context)
+        transaction = @transactions[trace_id(context)]
+        transaction ? transaction["signals"].dup : {}
+      end
+
+      def query_signals(payload, duration)
+        signals = {}
+        signals["slow_query"] = 1 if duration >= @config.apm_slow_query_threshold_ms
+        signals["long_transaction"] = 1 if long_transaction?(payload, duration)
+        error_class = (payload["error_class"] || payload[:error_class]).to_s
+        signals["connection_error"] = 1 if error_class =~ /Connection|NoDatabase|Adapter/i
+        signals["deadlock"] = 1 if error_class =~ /Deadlock/i
+        signals
+      end
+
+      def long_transaction?(payload, duration)
+        operation = (payload["operation"] || payload[:operation]).to_s
+        name = (payload["name"] || payload[:name]).to_s
+        transaction = operation == "BEGIN" || operation == "COMMIT" || name =~ /TRANSACTION/i
+        transaction && duration >= @config.apm_long_transaction_threshold_ms
+      end
+
+      def dimensions_for(type, payload)
+        names = case type
+                when "request" then %w(route method)
+                when "query" then %w(adapter operation table fingerprint normalized_query name cached role shard source)
+                when "job" then %w(kind class queue status)
+                else []
+                end
+        names.each_with_object({}) do |name, result|
+          value = payload[name] || payload[name.to_sym]
+          next if value.nil? || value.to_s.empty?
+
+          result[name] = dimension_value(type, name, value)
+        end
+      end
+
+      def dimension_value(type, name, value)
+        return value == true if name == "cached"
+        return value.to_i if type == "request" && name == "status"
+
+        text = value.to_s
+        text = normalize_route(text) if name == "route"
+        text.bytesize > 512 ? text.byteslice(0, 512) : text
+      end
+
+      def normalize_route(route)
+        route.split("/").map do |segment|
+          segment =~ /\A\d+\z/ || segment =~ /\A[0-9a-f]{8}-[0-9a-f-]{27,}\z/i ? ":id" : segment
+        end.join("/")
+      end
+
+      def metric_key(type, dimensions)
+        type + "|" + dimensions.keys.sort.map { |key| "#{key}=#{dimensions[key]}" }.join("|")
+      end
+
+      def component_category(type, payload)
+        return "database" if type == "query"
+        return "view" if type == "request" && payload["kind"].to_s == "view"
+        return "cache" if type == "cache"
+        return "queue" if type == "job"
+
+        nil
+      end
+
+      def error?(type, payload)
+        return (payload["status"] || payload[:status]).to_i >= 500 if type == "request"
+        return (payload["status"] || payload[:status]).to_s == "failed" if type == "job"
+
+        !(payload["error_class"] || payload[:error_class]).to_s.empty?
+      end
+
+      def trace_id(context)
+        (context["trace_id"] || context[:trace_id]).to_s
+      end
+
+      def increment_signal(transaction, name)
+        transaction["signals"][name] ||= 0
+        transaction["signals"][name] += 1
+      end
+
+      def merge_numeric(left, right)
+        hash(right).each do |key, value|
+          left[key.to_s] ||= 0.0
+          left[key.to_s] += non_negative(value)
+        end
+        left
+      end
+
+      def drain_locked
+        metrics = @groups.values.map(&:to_h)
+        if metrics.empty?
+          @transactions = {}
+          return []
+        end
+
+        dropped = @dropped_groups
+        @groups = {}
+        @transactions = {}
+        @observations = 0
+        @dropped_groups = 0
+        batches = []
+        metrics.each_slice(@config.apm_batch_size) do |slice|
+          batches << {"metrics" => slice, "dropped_groups" => batches.empty? ? dropped : 0}
+        end
+        batches
+      end
+
+      def non_negative(value)
+        number = value.to_f
+        number < 0.0 ? 0.0 : number
+      rescue StandardError
+        0.0
+      end
+
+      def hash(value)
+        value.is_a?(Hash) ? value : {}
+      end
+    end
+  end
+end

data/lib/chronos/application/capture_telemetry.rb

@@ -2,10 +2,10 @@ module Chronos
   module Application
     # Coordinates bounded non-exception telemetry capture.
     #
-    # @responsibility Build, sanitize, serialize, and enqueue integration events.
-    # @motivation Keep Rails notification policy outside transport and domain objects.
-    # @limits It handles request, query, job, and cache events only.
-    # @collaborators TelemetryEvent, TelemetrySerializer, and DeliveryPipeline.
+    # @responsibility Aggregate APM observations or serialize and enqueue integration events.
+    # @motivation Keep framework policy and local batching outside transport and domain objects.
+    # @limits It handles request, query, job, cache, and metric-batch events only.
+    # @collaborators ApmAggregator, TelemetryEvent, TelemetrySerializer, and DeliveryPipeline.
     # @thread_safety Calls own event state and may execute concurrently.
     # @compatibility Ruby 2.2.10 through Ruby 2.6; independent of Rails.
     # @example
@@ -20,18 +20,58 @@ module Chronos
         @serializer = options[:serializer] || Core::TelemetrySerializer.new(
           config, nil, :max_payload_size => proc { @delivery_pipeline.max_payload_size }
         )
+        @aggregator = options[:aggregator] || ApmAggregator.new(config)
       end
 
       def call(event_type, payload = {}, context = {})
         return false unless @config.enabled_for_environment?
         return false unless @delivery_pipeline.capture_allowed?(event_type)
 
+        aggregate = aggregation_enabled?
+        batches = aggregate ? @aggregator.record(event_type, payload, context) : []
+        return enqueue_batches(batches) if aggregate && aggregate_type?(event_type)
+
         event = Core::TelemetryEvent.new(event_type, payload, context)
-        @delivery_pipeline.enqueue(@serializer.call(event))
+        delivered = @delivery_pipeline.enqueue(@serializer.call(event))
+        enqueue_batches(batches)
+        delivered
       rescue StandardError => error
         @logger.warn("Chronos telemetry capture failed: #{error.class}")
         false
       end
+
+      def flush
+        enqueue_batches(@aggregator.flush)
+      rescue StandardError => error
+        @logger.warn("Chronos APM flush failed: #{error.class}")
+        false
+      end
+
+      def diagnostics
+        @aggregator.diagnostics
+      rescue StandardError
+        {}
+      end
+
+      private
+
+      def aggregate_type?(event_type)
+        ApmAggregator::METRIC_TYPES.include?(event_type.to_s)
+      end
+
+      def aggregation_enabled?
+        @config.apm_enabled && @delivery_pipeline.event_enabled?("metric_batch")
+      end
+
+      def enqueue_batches(batches)
+        results = Array(batches).map do |batch|
+          next false unless @delivery_pipeline.event_enabled?("metric_batch")
+
+          event = Core::TelemetryEvent.new("metric_batch", batch)
+          @delivery_pipeline.enqueue(@serializer.call(event))
+        end
+        results.empty? || results.all?
+      end
     end
   end
 end

data/lib/chronos/application/delivery_pipeline.rb

@@ -67,6 +67,13 @@ module Chronos
         false
       end
 
+      def event_enabled?(event_type)
+        @remote_configuration.delivery_enabled?(event_type)
+      rescue StandardError => error
+        diagnose(error)
+        false
+      end
+
       def max_payload_size
         @remote_configuration.max_payload_size
       end

data/lib/chronos/application/remote_configuration.rb

@@ -13,7 +13,7 @@ module Chronos
     # @errors Invalid documents are ignored and return false.
     # @performance Validation is bounded by configured document and list limits.
     class RemoteConfiguration
-      SUPPORTED_EVENT_TYPES = ["exception", "request", "query", "job", "cache"].freeze
+      SUPPORTED_EVENT_TYPES = ["exception", "request", "query", "job", "cache", "metric_batch"].freeze
       MAX_IGNORED_FINGERPRINTS = 100
       MAX_FINGERPRINT_BYTES = 256
       MIN_PAYLOAD_SIZE = 256
@@ -63,6 +63,11 @@ module Chronos
         snapshot["enabled_event_types"].include?(event_type.to_s)
       end
 
+      def delivery_enabled?(event_type)
+        values = snapshot
+        !values["kill_switch"] && values["enabled_event_types"].include?(event_type.to_s)
+      end
+
       def ignored_fingerprint?(fingerprint)
         snapshot["ignored_fingerprints"].include?(fingerprint.to_s)
       end

data/lib/chronos/configuration/apm_validation.rb

@@ -0,0 +1,60 @@
+module Chronos
+  module Internal
+    # Validates bounded APM aggregation, histogram, and detector settings.
+    #
+    # @responsibility Return configuration errors for version 0.7 APM options.
+    # @motivation Keep the general configuration validator focused and maintainable.
+    # @limits It validates shape and bounds but does not allocate aggregator state.
+    # @collaborators Chronos::Configuration predicates and attributes.
+    # @thread_safety Reads one mutable configuration instance without shared state.
+    # @compatibility Ruby 2.2.10 through Ruby 2.6.
+    # @errors Invalid values become messages and never raise from validation.
+    module ApmConfigurationValidation
+      private
+
+      def apm_errors
+        errors = apm_capacity_errors
+        errors.concat(apm_threshold_errors)
+        errors << "apm_enabled must be true or false" unless [true, false].include?(apm_enabled)
+        unless increasing_positive_numbers?(apm_histogram_buckets)
+          errors << "apm_histogram_buckets must contain increasing positive numbers"
+        end
+        errors
+      end
+
+      def apm_capacity_errors
+        errors = []
+        errors << "apm_max_groups must be a positive integer" unless positive_integer?(apm_max_groups)
+        errors << "apm_flush_count must be a positive integer" unless positive_integer?(apm_flush_count)
+        unless apm_batch_size.is_a?(Integer) && apm_batch_size >= 1 && apm_batch_size <= 50
+          errors << "apm_batch_size must be between 1 and 50"
+        end
+        unless positive_integer?(apm_max_queries_per_request)
+          errors << "apm_max_queries_per_request must be a positive integer"
+        end
+        errors
+      end
+
+      def apm_threshold_errors
+        errors = []
+        unless positive_number?(apm_slow_query_threshold_ms)
+          errors << "apm_slow_query_threshold_ms must be greater than zero"
+        end
+        unless positive_number?(apm_long_transaction_threshold_ms)
+          errors << "apm_long_transaction_threshold_ms must be greater than zero"
+        end
+        unless apm_n_plus_one_threshold.is_a?(Integer) && apm_n_plus_one_threshold >= 2
+          errors << "apm_n_plus_one_threshold must be an integer greater than or equal to 2"
+        end
+        errors
+      end
+
+      def increasing_positive_numbers?(values)
+        return false unless values.is_a?(Array) && !values.empty? && values.length <= 19
+        return false unless values.all? { |value| positive_number?(value) }
+
+        values.each_cons(2).all? { |left, right| right > left }
+      end
+    end
+  end
+end

data/lib/chronos/configuration.rb

@@ -1,5 +1,6 @@
 require "uri"
 require "chronos/configuration/validation"
+require "chronos/configuration/apm_validation"
 
 module Chronos
   # Mutable configuration used only while the Chronos agent is being set up.
@@ -21,6 +22,7 @@ module Chronos
   #   snapshot = config.snapshot
   class Configuration
     include Internal::ConfigurationValidation
+    include Internal::ApmConfigurationValidation
     DEFAULT_BLOCKLIST_KEYS = %w(
       password password_confirmation passwd secret api_key apikey authorization
       token access_token refresh_token private_key client_secret cookie set-cookie
@@ -40,7 +42,11 @@ module Chronos
       :remote_configuration, :remote_config_max_bytes, :sampling_rate,
       :enabled_event_types, :max_remote_send_interval, :context_store,
       :breadcrumb_capacity, :breadcrumb_max_bytes, :rails_enabled,
-      :rails_capture_in_console, :rails_capture_in_test, :rails_capture_user_agent
+      :rails_capture_in_console, :rails_capture_in_test, :rails_capture_user_agent,
+      :apm_enabled, :apm_max_groups, :apm_flush_count, :apm_batch_size,
+      :apm_max_queries_per_request, :apm_slow_query_threshold_ms,
+      :apm_long_transaction_threshold_ms, :apm_n_plus_one_threshold,
+      :apm_histogram_buckets
     ].freeze
 
     attr_accessor(*ATTRIBUTES)
@@ -50,6 +56,7 @@ module Chronos
       initialize_privacy_defaults
       initialize_resilience_defaults
       initialize_rails_defaults
+      initialize_apm_defaults
     end
 
     def snapshot
@@ -78,6 +85,7 @@ module Chronos
       errors.concat(resilience_errors)
       errors.concat(privacy_errors)
       errors.concat(context_errors)
+      errors.concat(apm_errors)
       errors
     end
 
@@ -116,6 +124,18 @@ module Chronos
       @rails_capture_user_agent = false
     end
 
+    def initialize_apm_defaults
+      @apm_enabled = true
+      @apm_max_groups = 200
+      @apm_flush_count = 100
+      @apm_batch_size = 50
+      @apm_max_queries_per_request = 100
+      @apm_slow_query_threshold_ms = 500.0
+      @apm_long_transaction_threshold_ms = 1000.0
+      @apm_n_plus_one_threshold = 5
+      @apm_histogram_buckets = [5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0]
+    end
+
     def initialize_privacy_defaults
       @blocklist_keys = DEFAULT_BLOCKLIST_KEYS.dup
       @allowlist_keys = []
@@ -135,7 +155,7 @@ module Chronos
       @remote_configuration = true
       @remote_config_max_bytes = 4096
       @sampling_rate = 1.0
-      @enabled_event_types = ["exception", "request", "query", "job", "cache"]
+      @enabled_event_types = ["exception", "request", "query", "job", "cache", "metric_batch"]
       @max_remote_send_interval = 60.0
     end