scout_apm 1.1.0.pre1 → 1.2.0.pre1

data/lib/scout_apm/metric_stats.rb

@@ -1,11 +1,14 @@
 # Stats that are associated with each instrumented method.
-class ScoutApm::MetricStats
+module ScoutApm
+class MetricStats
   attr_accessor :call_count
   attr_accessor :min_call_time
   attr_accessor :max_call_time
   attr_accessor :total_call_time
   attr_accessor :total_exclusive_time
   attr_accessor :sum_of_squares
+  attr_accessor :queue
+  attr_accessor :latency
 
   def initialize(scoped = false)
     @scoped = scoped
@@ -17,16 +20,22 @@ class ScoutApm::MetricStats
     self.sum_of_squares = 0.0
   end
 
-  def update!(call_time,exclusive_time)
+  # Note, that you must include exclusive_time if you wish to set
+  # extra_metrics. A two argument use of this method won't do that.
+  def update!(call_time, exclusive_time=call_time, extra_metrics={})
     # If this metric is scoped inside another, use exclusive time for min/max and sum_of_squares. Non-scoped metrics
     # (like controller actions) track the total call time.
     t = (@scoped ? exclusive_time : call_time)
     self.min_call_time = t if self.call_count == 0 or t < min_call_time
     self.max_call_time = t if self.call_count == 0 or t > max_call_time
-    self.call_count +=1
+    self.call_count += 1
     self.total_call_time += call_time
     self.total_exclusive_time += exclusive_time
     self.sum_of_squares += (t * t)
+    if extra_metrics
+      self.queue = extra_metrics[:queue] if extra_metrics[:queue]
+      self.latency = extra_metrics[:latency] if extra_metrics[:latency]
+    end
     self
   end
 
@@ -52,4 +61,5 @@ class ScoutApm::MetricStats
     # uri, context
     ScoutApm::AttributeArranger.call(self, json_attributes)
   end
-end # class MetricStats
+end
+end

data/lib/scout_apm/request_manager.rb

@@ -0,0 +1,26 @@
+# Request manager handles the threadlocal variable that holds the current
+# request. If there isn't one, then create one
+
+module ScoutApm
+  class RequestManager
+    def self.lookup
+      find || create
+    end
+
+    # Get the current Thread local, and detecting, and not returning a stale request
+    def self.find
+      req = Thread.current[:scout_request]
+
+      if req && req.recorded?
+        nil
+      else
+        req
+      end
+    end
+
+    # Create a new TrackedRequest object for this thread
+    def self.create
+      Thread.current[:scout_request] = TrackedRequest.new
+    end
+  end
+end

data/lib/scout_apm/request_queue_time.rb

@@ -0,0 +1,54 @@
+module ScoutApm
+  class RequestQueueTime < LayerConverterBase
+    HEADERS = %w(X-Queue-Start X-Request-Start X-QUEUE-START X-REQUEST-START x-queue-start x-request-start)
+
+    # Headers is a hash of request headers.  In Rails, request.headers would be appropriate
+    def initialize(request)
+      super(request)
+      @headers = request.headers
+    end
+
+    def call
+      return {} unless headers
+
+      raw_start = locate_timestamp
+      return {} unless raw_start
+
+      parsed_start = parse(raw_start)
+      return {} unless parsed_start
+
+      request_start = root_layer.start_time
+      queue_time = (request_start - parsed_start).to_f
+
+      meta = MetricMeta.new("QueueTime/Request", {:scope => scope_layer.legacy_metric_name})
+      stat = MetricStats.new(true)
+      stat.update!(queue_time)
+
+      { meta => stat }
+    end
+
+    private
+
+    attr_reader :headers
+
+    # Looks through the possible headers with this data, and extracts the raw
+    # value of the header
+    # Returns nil if not found
+    def locate_timestamp
+      return nil unless headers
+
+      header = HEADERS.find { |candidate| headers[candidate] }
+      if header
+        data = headers[header]
+        data.to_s.gsub(/(t=|\.)/, '')
+      else
+        nil
+      end
+    end
+
+    # Returns a timestamp in fractional seconds since epoch
+    def parse(time_string)
+      Time.at("#{time_string[0,10]}.#{time_string[10,13]}".to_f)
+    end
+  end
+end

data/lib/scout_apm/serializers/payload_serializer.rb

@@ -6,7 +6,14 @@ module ScoutApm
         if ScoutApm::Agent.instance.config.value("report_format") == 'json'
           ScoutApm::Serializers::PayloadSerializerToJson.serialize(metadata, metrics, slow_transactions)
         else
-          Marshal.dump(:metadata => metadata, :metrics => metrics, :slow_transactions => slow_transactions)
+          metadata = metadata.dup
+          metadata.default_proc = nil
+
+          metrics = metrics.dup
+          metrics.default_proc = nil
+          Marshal.dump(:metadata          => metadata,
+                       :metrics           => metrics,
+                       :slow_transactions => slow_transactions)
         end
       end
 

data/lib/scout_apm/serializers/payload_serializer_to_json.rb

@@ -5,6 +5,7 @@ module ScoutApm
         def serialize(metadata, metrics, slow_transactions)
           rearranged_metrics = rearrange_the_metrics(metrics)
           rearranged_slow_transactions = rearrange_the_slow_transactions(slow_transactions)
+          metadata.merge!({:payload_version => 2})
           jsonify_hash({:metadata => metadata, :metrics => rearranged_metrics, :slow_transactions => rearranged_slow_transactions})
         end
 

data/lib/scout_apm/slow_transaction.rb

@@ -16,6 +16,9 @@ module ScoutApm
     attr_reader :prof
     attr_reader :raw_prof
 
+    # TODO: Move this out of SlowTransaction, it doesn't have much to do w/
+    # slow trans other than being a piece of data that ends up in it.
+    #
     # Given a call stack, generates a filtered backtrace that:
     # * Limits to the app/models, app/controllers, or app/views directories
     # * Limits to 5 total callers

data/lib/scout_apm/store.rb

@@ -1,220 +1,152 @@
-
-# The store encapsolutes the logic that (1) saves instrumented data by Metric name to memory and (2) maintains a stack (just an Array)
-# of instrumented methods that are being called. It's accessed via +ScoutApm::Agent.instance.store+.
+# Stores one or more minute's worth of Metrics/SlowTransactions in local ram.
+# When informed to by the background worker, it pushes the in-ram metrics off to
+# the layaway file for cross-process aggregation.
 module ScoutApm
   class Store
+    # A hash of reporting periods. { StoreReportingPeriodTimestamp => StoreReportingPeriod }
+    attr_reader :reporting_periods
 
-    # Limits the size of the metric hash to prevent a metric explosion.
-    MAX_SIZE = 1000
+    def initialize
+      @reporting_periods = Hash.new { |h,k| h[k] = StoreReportingPeriod.new(k) }
+    end
 
-    # Limit the number of slow transactions that we store metrics with to prevent writing too much data to the layaway file if there are are many processes and many slow slow_transactions.
-    MAX_SLOW_TRANSACTIONS_TO_STORE_METRICS = 10
+    def current_timestamp
+      StoreReportingPeriodTimestamp.new
+    end
 
-    attr_accessor :metric_hash
-    attr_accessor :transaction_hash
-    attr_accessor :stack
-    attr_accessor :slow_transactions # array of slow transaction slow_transactions
-    attr_reader :slow_transaction_lock
+    # Save newly collected metrics
+    def track!(metrics, options={})
+      reporting_periods[current_timestamp].merge_metrics!(metrics)
+    end
 
-    def initialize
-      @metric_hash = Hash.new
-      # Stores aggregate metrics for the current transaction. When the transaction is finished, metrics
-      # are merged with the +metric_hash+.
-      @transaction_hash = Hash.new
-      @stack = Array.new
-      # ensure background thread doesn't manipulate transaction sample while the store is.
-      @slow_transaction_lock = Mutex.new
-      @slow_transactions = Array.new
+    def track_one!(type, name, value, options={})
+      meta = MetricMeta.new("#{type}/#{name}")
+      stat = MetricStats.new(false)
+      stat.update!(value)
+      track!({meta => stat}, options)
     end
 
-    # Called when the last stack item completes for the current transaction to clear
-    # for the next run.
-    def reset_transaction!
-      Thread::current[:scout_apm_ignore_transaction] = nil
-      Thread::current[:scout_apm_scope_name] = nil
-      @transaction_hash = Hash.new
-      @stack = Array.new
+    # Save a new slow transaction
+    def track_slow_transaction!(slow_transaction)
+      reporting_periods[current_timestamp].merge_slow_transactions!(slow_transaction)
     end
 
-    def ignore_transaction!
-      Thread::current[:scout_apm_ignore_transaction] = true
+    # Take each completed reporting_period, and write it to the layaway passed
+    def write_to_layaway(layaway)
+      reporting_periods.select { |time, rp| time.timestamp < current_timestamp.timestamp}.
+                        each { |time, reporting_period|
+                               layaway.add_reporting_period(time, reporting_period)
+                               reporting_periods.delete(time)
+                             }
     end
+  end
+
+  # A timestamp, normalized to the beginning of a minute. Used as a hash key to
+  # bucket metrics into per-minute groups
+  class StoreReportingPeriodTimestamp
+    attr_reader :timestamp
 
-    # Called at the start of Tracer#instrument:
-    # (1) Either finds an existing MetricStats object in the metric_hash or
-    # initialize a new one. An existing MetricStats object is present if this +metric_name+ has already been instrumented.
-    # (2) Adds a StackItem to the stack. This StackItem is returned and later used to validate the item popped off the stack
-    # when an instrumented code block completes.
-    def record(metric_name)
-      item = ScoutApm::StackItem.new(metric_name)
-      stack << item
-      item
+    def initialize(time=Time.now)
+      @raw_time = time.utc # The actual time passed in. Store it so we can to_s it without reparsing a timestamp
+      @timestamp = @raw_time.to_i - @raw_time.sec # The normalized time (integer) to compare by
     end
 
-    # Options:
-    # * :scope - If specified, sets the sub-scope for the metric. We allow additional scope level. This is used
-    # * uri - the request uri
-    def stop_recording(sanity_check_item, options={})
-      item = stack.pop
-      stack_empty = stack.empty?
+    def to_s
+      @raw_time.iso8601
+    end
 
-      # if ignoring the transaction, the item is popped but nothing happens.
-      if Thread::current[:scout_apm_ignore_transaction]
-        return
-      end
+    def eql?(o)
+      timestamp.eql?(o.timestamp)
+    end
 
-      # unbalanced stack check - unreproducable cases have seen this occur. when it does, sets a Thread variable
-      # so we ignore further recordings. +Store#reset_transaction!+ resets this.
-      if item != sanity_check_item
-        ScoutApm::Agent.instance.logger.warn "Scope [#{Thread::current[:scout_apm_scope_name]}] Popped off stack: #{item.inspect} Expected: #{sanity_check_item.inspect}. Aborting."
-        ignore_transaction!
-        return
-      end
+    def hash
+      timestamp.hash
+    end
 
-      duration = Time.now - item.start_time
-      if last = stack.last
-        last.children_time += duration
-      end
+    def age_in_seconds
+      Time.now.to_i - timestamp
+    end
+  end
 
-      meta = ScoutApm::MetricMeta.new(item.metric_name, :desc => options[:desc])
-      meta.scope = nil if stack_empty
+  # One period of Storage. Typically 1 minute
+  class StoreReportingPeriod
+    # A hash of { MetricMeta => MetricStat }
+    # This holds metrics for specific parts of the application.
+    # "Controller/user/index", "ActiveRecord/SQL/users/find", "View/users/_gravatar" and similar.
+    #
+    # If over the course of a minute a metric is called more than once (very likely), it will be
+    # combined with the others of the same type, and summed/calculated.  The merging logic is in
+    # MetricStats
+    #
+    # Use the accessor function `metrics_payload` for most uses. It includes the calculated aggregate values
+    attr_reader :metrics
 
-      # add backtrace for slow calls ... how is exclusive time handled?
-      if duration > ScoutApm::SlowTransaction::BACKTRACE_THRESHOLD and !stack_empty
-        meta.extra = {:backtrace => ScoutApm::SlowTransaction.backtrace_parser(caller)}
-      end
+    # An array of SlowTransaction objects
+    attr_reader :slow_transactions
 
-      stat = transaction_hash[meta] || ScoutApm::MetricStats.new(!stack_empty)
-      stat.update!(duration,duration-item.children_time)
-      transaction_hash[meta] = stat if store_metric?(stack_empty)
-
-      # Uses controllers as the entry point for a transaction. Otherwise, stats are ignored.
-      if stack_empty and meta.metric_name.match(/\AController\//)
-        aggs = aggregate_calls(transaction_hash.dup,meta)
-        store_slow(options[:uri], transaction_hash.dup.merge(aggs), meta, stat)
-        # deep duplicate
-        duplicate = aggs.dup
-        duplicate.each_pair do |k,v|
-          duplicate[k.dup] = v.dup
-        end
-        merge_metrics(duplicate.merge({meta.dup => stat.dup})) # aggregrates + controller
-      end
+    # A StoreReportingPeriodTimestamp representing the time that this
+    # collection of metrics is for
+    attr_reader :timestamp
+
+    def initialize(timestamp)
+      @metrics = Hash.new
+      @slow_transactions = Array.new
+      @timestamp = timestamp
     end
 
-    # TODO - Move more logic to SlowTransaction
-    #
-    # Limits the size of the transaction hash to prevent a large transactions. The final item on the stack
-    # is allowed to be stored regardless of hash size to wrapup the transaction sample w/the parent metric.
-    def store_metric?(stack_empty)
-      transaction_hash.size < ScoutApm::SlowTransaction::MAX_SIZE or stack_empty
-    end
-
-    # Returns the top-level category names used in the +metrics+ hash.
-    def categories(metrics)
-      cats = Set.new
-      metrics.keys.each do |meta|
-        next if meta.scope.nil? # ignore controller
-        if match=meta.metric_name.match(/\A([\w]+)\//)
-          cats << match[1]
-        end
-      end # metrics.each
-      cats
-    end
-
-    # Takes a metric_hash of calls and generates aggregates for ActiveRecord and View calls.
-    def aggregate_calls(metrics,parent_meta)
-      categories = categories(metrics)
-      aggregates = {}
-      categories.each do |cat|
-        agg_meta=ScoutApm::MetricMeta.new("#{cat}/all")
-        agg_meta.scope = parent_meta.metric_name
-        agg_stats = ScoutApm::MetricStats.new
-        metrics.each do |meta,stats|
-          if meta.metric_name =~ /\A#{cat}\//
-            agg_stats.combine!(stats)
-          end
-        end # metrics.each
-        aggregates[agg_meta] = agg_stats unless agg_stats.call_count.zero?
-      end # categories.each
-      aggregates
-    end
-
-    SLOW_TRANSACTION_THRESHOLD = 2
-
-    # Stores slow transactions. This will be sent to the server.
-    def store_slow(uri, transaction_hash, parent_meta, parent_stat, options = {})
-      @slow_transaction_lock.synchronize do
-        if parent_stat.total_call_time >= SLOW_TRANSACTION_THRESHOLD
-          slow_transaction = ScoutApm::SlowTransaction.new(uri,
-                                                           parent_meta.metric_name,
-                                                           parent_stat.total_call_time,
-                                                           transaction_hash.dup,
-                                                           ScoutApm::Context.current,
-                                                           Time.now,
-                                                           Thread::current[:scout_apm_prof])
-          @slow_transactions.push(slow_transaction)
-          ScoutApm::Agent.instance.logger.debug "Slow transaction sample added. [URI: #{uri}] [Context: #{ScoutApm::Context.current.to_hash}] Array Size: #{@slow_transactions.size}"
-        end
-      end
+    #################################
+    # Add metrics as they are recorded
+    #################################
+    def merge_metrics!(metrics)
+      @metrics.merge!(metrics) { |key, old_stat, new_stat| old_stat.combine!(new_stat) }
+      self
     end
 
-    # Finds or creates the metric w/the given name in the metric_hash, and updates the time. Primarily used to
-    # record sampled metrics. For instrumented methods, #record and #stop_recording are used.
-    #
-    # Options:
-    # :scope => If provided, overrides the default scope.
-    # :exclusive_time => Sets the exclusive time for the method. If not provided, uses +call_time+.
-    def track!(metric_name, call_time, options = {})
-      meta = ScoutApm::MetricMeta.new(metric_name)
-      meta.scope = options[:scope] if options.has_key?(:scope)
-      stat = metric_hash[meta] || ScoutApm::MetricStats.new
-      stat.update!(call_time,options[:exclusive_time] || call_time)
-      metric_hash[meta] = stat
-    end
-
-    # Combines old and current data
-    def merge_data(old_data)
-      {
-        :metrics           => merge_metrics(old_data[:metrics]),
-        :slow_transactions => merge_slow_transactions(old_data[:slow_transactions])
-      }
-    end
-
-    # Merges old and current data, clears the current in-memory metric hash, and returns
-    # the merged data
-    def merge_data_and_clear(old_data)
-      merged = merge_data(old_data)
-      self.metric_hash =  {}
-      # TODO - is this lock needed?
-      @slow_transaction_lock.synchronize do
-        self.slow_transactions = []
-      end
-      merged
+    def merge_slow_transactions!(slow_transactions)
+      @slow_transactions += Array(slow_transactions)
+      self
     end
 
-    def merge_metrics(old_metrics)
-      old_metrics.each do |old_meta,old_stats|
-        if stats = metric_hash[old_meta]
-          metric_hash[old_meta] = stats.combine!(old_stats)
-        elsif metric_hash.size < MAX_SIZE
-          metric_hash[old_meta] = old_stats
-        end
-      end
-      metric_hash
-    end
-
-    # Merges slow_transactions together, removing transaction sample metrics from slow_transactions if the > MAX_SLOW_TRANSACTIONS_TO_STORE_METRICS
-    def merge_slow_transactions(old_slow_transactions)
-      # need transaction lock here?
-      self.slow_transactions += old_slow_transactions
-      if trim_slow_transactions = self.slow_transactions[MAX_SLOW_TRANSACTIONS_TO_STORE_METRICS..-1]
-        ScoutApm::Agent.instance.logger.debug "Trimming metrics from #{trim_slow_transactions.size} slow_transactions."
-        i = MAX_SLOW_TRANSACTIONS_TO_STORE_METRICS
-        trim_slow_transactions.each do |sample|
-          self.slow_transactions[i] = sample.clear_metrics!
+    #################################
+    # Retrieve Metrics for reporting
+    #################################
+    def metrics_payload
+      aggregate_metrics
+    end
+
+    def slow_transactions_payload
+      @slow_transactions
+    end
+
+    private
+
+    # We can't aggregate CPU, Memory, Capacity, or Controller, so pass through these metrics directly
+    # TODO: Figure out a way to not have this duplicate what's in Samplers, and also on server's ingest
+    PASSTHROUGH_METRICS = ["CPU", "Memory", "Instance", "Controller"]
+
+    # Calculate any aggregate metrics necessary.
+    #
+    # A hash of { MetricMeta => MetricStat }
+    # This represents the aggregate metrics over the course of the minute.
+    # "ActiveRecord/all", "View/all", "HTTP/all" and similar
+    def aggregate_metrics
+      hsh = Hash.new {|h,k| h[k] = MetricStats.new }
+
+      @metrics.inject(hsh) do |result, (meta, stat)|
+        if PASSTHROUGH_METRICS.include?(meta.type) # Leave as-is, don't attempt to combine
+          hsh[meta] = stat
+        elsif meta.type == "Errors" # Sadly special cased, we want both raw and aggregate values
+          hsh[meta] = stat
+          agg_meta = MetricMeta.new("Errors/Request", :scope => meta.scope)
+          hsh[agg_meta].combine!(stat)
+        else # Combine down to a single /all key
+          agg_meta = MetricMeta.new("#{meta.type}/all", :scope => meta.scope)
+          hsh[agg_meta].combine!(stat)
         end
+
+        hsh
       end
-      self.slow_transactions
     end
-  end # class Store
+  end
 end
+