scout_apm 1.5.5 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca18811f4b9adad075737c7321340025345f1c14
-  data.tar.gz: ca2873e587e8d70fc1a3bffe564b59cde10b0f85
+  metadata.gz: da7b4f6ec7fe62d8ee8243439e1c98404d2257a5
+  data.tar.gz: b3a4f52a7d88731fa0c603aef7b0a236535edb82
 SHA512:
-  metadata.gz: 7a4e009a252a824a6e22bde107ad671e6c017767ad6d96960ecdf056e0ccef3761e6d8d9abe984ec533246c8f34ccba71cc53ce4f7e3012dc2fb2b7173104578
-  data.tar.gz: d45a047c05341ef7da71c1b38a370d78f3e854a54b27ecac7a1bd33a37fd88bb4b30bd1c7d7fd2f897999a1b85e06d2e62437204d115152bd84470f7a36e9497
+  metadata.gz: c74c1effbb0846908bd6f61f6bfc1976cac80fa40ec789974c45c2e4c2e04cc2043967480b05e1541989668f096644c7e7bfa51feb4eea2825696b542142cf99
+  data.tar.gz: 3bb6ab6fbe7847c0c4cc952c23109189d03bad9f76a34afdde6e473c014de967a8d4e0c707ec766435d876f5346eda8bab7721a8167d6341932a794873cf86e3

data/CHANGELOG.markdown

@@ -1,3 +1,11 @@
+# 1.6.0
+
+* Dynamic algorithm for selecting when to collect traces. Now, we will collect a
+  more complete cross-section of your application's performance, dynamically
+  tuned as your application runs.
+* Record and report 95th percentiles for each action
+* A variety of bug fixes
+
 # 1.5.5
 
 * Handle backslash escaped quotes inside mysql strings.

data/lib/scout_apm.rb

@@ -85,6 +85,7 @@ require 'scout_apm/instruments/rails_router'
 require 'scout_apm/instruments/sinatra'
 require 'scout_apm/instruments/process/process_cpu'
 require 'scout_apm/instruments/process/process_memory'
+require 'scout_apm/instruments/percentile_sampler'
 
 require 'scout_apm/app_server_load'
 
@@ -118,9 +119,11 @@ require 'scout_apm/metric_stats'
 require 'scout_apm/slow_transaction'
 require 'scout_apm/slow_job_record'
 require 'scout_apm/slow_item_set'
+require 'scout_apm/scored_item_set'
 require 'scout_apm/slow_request_policy'
 require 'scout_apm/slow_job_policy'
 require 'scout_apm/job_record'
+require 'scout_apm/request_histograms'
 
 require 'scout_apm/capacity'
 require 'scout_apm/attribute_arranger'

data/lib/scout_apm/agent.rb

@@ -20,6 +20,13 @@ module ScoutApm
     attr_reader :slow_request_policy
     attr_reader :slow_job_policy
 
+    # Histogram of the cumulative requests since the start of the process
+    attr_reader :request_histograms
+
+    # Histogram of the requests, distinct by reporting period (minute)
+    # { StoreReportingPeriodTimestamp => RequestHistograms }
+    attr_reader :request_histograms_by_time
+
     # All access to the agent is thru this class method to ensure multiple Agent instances are not initialized per-Ruby process.
     def self.instance(options = {})
       @@instance ||= self.new(options)
@@ -32,14 +39,16 @@ module ScoutApm
       @started = false
       @options ||= options
       @config = ScoutApm::Config.new(options[:config_path])
+
+      @slow_request_policy = ScoutApm::SlowRequestPolicy.new
       @slow_job_policy = ScoutApm::SlowJobPolicy.new
+      @request_histograms = ScoutApm::RequestHistograms.new
+      @request_histograms_by_time = Hash.new { |h, k| h[k] = ScoutApm::RequestHistograms.new }
 
       @store          = ScoutApm::Store.new
       @layaway        = ScoutApm::Layaway.new
       @metric_lookup  = Hash.new
 
-      @slow_request_policy = ScoutApm::SlowRequestPolicy.new
-
       @capacity       = ScoutApm::Capacity.new
       @installed_instruments = []
     end
@@ -103,10 +112,10 @@ module ScoutApm
 
       load_instruments if should_load_instruments?(options)
 
-      @samplers = [
-        ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors, logger),
-        ScoutApm::Instruments::Process::ProcessMemory.new(logger)
-      ]
+      [ ScoutApm::Instruments::Process::ProcessCpu.new(environment.processors, logger),
+        ScoutApm::Instruments::Process::ProcessMemory.new(logger),
+        ScoutApm::Instruments::PercentileSampler.new(logger, 95),
+      ].each { |s| store.add_sampler(s) }
 
       app_server_load_hook
 
@@ -218,16 +227,19 @@ module ScoutApm
       @background_worker = ScoutApm::BackgroundWorker.new
       @background_worker_thread = Thread.new do
         @background_worker.start {
-          # First, run periodic samplers. These should run once a minute,
-          # rather than per-request. "CPU Load" and similar.
-          run_samplers
-          capacity.process
-
           ScoutApm::Agent.instance.process_metrics
+          clean_old_percentiles
         }
       end
     end
 
+    def clean_old_percentiles
+      request_histograms_by_time.
+        keys.
+        select {|timestamp| timestamp.age_in_seconds > 60 * 10 }.
+        each {|old_timestamp| request_histograms_by_time.delete(old_timestamp) }
+    end
+
     # If we want to skip the app_server_check, then we must load it.
     def should_load_instruments?(options={})
       return true if options[:skip_app_server_check]
@@ -290,20 +302,6 @@ module ScoutApm
       environment.deploy_integration
     end
 
-    # TODO: Extract a proper class / registery for these. They don't really belong here
-    def run_samplers
-      @samplers.each do |sampler|
-        begin
-          result = sampler.run
-          store.track_one!(sampler.metric_type, sampler.metric_name, result) if result
-        rescue => e
-          logger.info "Error reading #{sampler.human_name}"
-          logger.debug e.message
-          logger.debug e.backtrace.join("\n")
-        end
-      end
-    end
-
     def app_server_missing?(options = {})
       !environment.app_server_integration(true).found? && !options[:skip_app_server_check]
     end

data/lib/scout_apm/agent/reporting.rb

@@ -50,7 +50,7 @@ module ScoutApm
           :platform      => "ruby",
         }
 
-        log_deliver(metrics, slow_transactions, metadata)
+        log_deliver(metrics, slow_transactions, metadata, slow_jobs)
 
         payload = ScoutApm::Serializers::PayloadSerializer.serialize(metadata, metrics, slow_transactions, jobs, slow_jobs)
         logger.debug("Payload: #{payload}")
@@ -62,7 +62,7 @@ module ScoutApm
         logger.debug e.backtrace
       end
 
-      def log_deliver(metrics, slow_transactions, metadata)
+      def log_deliver(metrics, slow_transactions, metadata, jobs_traces)
         total_request_count = metrics.
           select { |meta,stats| meta.metric_name =~ /\AController/ }.
           inject(0) {|sum, (_, stat)| sum + stat.call_count }
@@ -75,7 +75,12 @@ module ScoutApm
                             "Recorded across (unknown) processes"
                           end
 
-        logger.info "[#{Time.parse(metadata[:agent_time]).strftime("%H:%M")}] Delivering #{metrics.length} Metrics for #{total_request_count} requests and #{slow_transactions.length} Slow Transaction Traces, #{process_log_str}."
+        time_clause       = "[#{Time.parse(metadata[:agent_time]).strftime("%H:%M")}]"
+        metrics_clause    = "#{metrics.length} Metrics for #{total_request_count} requests"
+        slow_trans_clause = "#{slow_transactions.length} Slow Transaction Traces"
+        job_clause        = "#{jobs_traces.length} Job Traces"
+
+        logger.info "#{time_clause} Delivering #{metrics_clause} and #{slow_trans_clause} and #{job_clause}, #{process_log_str}."
         logger.debug("Metrics: #{metrics.pretty_inspect}\nSlowTrans: #{slow_transactions.pretty_inspect}\nMetadata: #{metadata.inspect.pretty_inspect}")
       end
 

data/lib/scout_apm/attribute_arranger.rb

@@ -7,6 +7,10 @@ module ScoutApm
         case attribute
         when Array
           attribute_hash[attribute[0]] = subject.send(attribute[1])
+        when :bucket
+          attribute_hash[attribute] = subject.bucket_type
+        when :name
+          attribute_hash[attribute] = subject.bucket_name
         when Symbol
           attribute_hash[attribute] = subject.send(attribute)
         end

data/lib/scout_apm/bucket_name_splitter.rb

@@ -1,15 +1,15 @@
 module ScoutApm
   module BucketNameSplitter
-    def bucket
+    def bucket_type
       split_metric_name(metric_name).first
     end
 
-    def name
+    def bucket_name
       split_metric_name(metric_name).last
     end
 
     def key
-      {:bucket => bucket, :name => name}
+      {:bucket => bucket_type, :name => bucket_name}
     end
 
     private

data/lib/scout_apm/config.rb

@@ -7,14 +7,15 @@ require 'scout_apm/environment'
 #
 # application_root - override the detected directory of the application
 # data_file        - override the default temporary storage location. Must be a location in a writable directory
-# hostname         - override the default hostname detection. Default varies by environment - either system hostname, or PAAS hostname
+# host             - override the default hostname detection. Default varies by environment - either system hostname, or PAAS hostname
+# direct_host      - override the default "direct" host. The direct_host bypasses the ingestion pipeline and goes directly to the webserver, and is primarily used for features under development.
 # key              - the account key with Scout APM. Found in Settings in the Web UI
 # log_file_path    - either a directory or "STDOUT".
 # log_level        - DEBUG / INFO / WARN as usual
 # monitor          - true or false.  False prevents any instrumentation from starting
 # name             - override the name reported to APM. This is the name that shows in the Web UI
 # uri_reporting    - 'path' or 'full_path' default is 'full_path', which reports URL params as well as the path.
-# report_format    - 'json' or 'marshal'. Marshal is currently the default; json processing is in beta
+# report_format    - 'json' or 'marshal'. Marshal is legacy and will be removed.
 #
 # Any of these config settings can be set with an environment variable prefixed
 # by SCOUT_ and uppercasing the key: SCOUT_LOG_LEVEL for instance.
@@ -23,6 +24,7 @@ module ScoutApm
   class Config
     DEFAULTS =  {
         'host'      => 'https://checkin.scoutapp.com',
+        'direct_host' => 'https://apm.scoutapp.com',
         'log_level' => 'info',
         'stackprof_interval' => 20000, # microseconds, 1000 = 1 millisecond, so 20k == 20 milliseconds
         'uri_reporting' => 'full_path',

data/lib/scout_apm/histogram.rb

@@ -56,6 +56,26 @@ module ScoutApm
       end
     end
 
+    # Given a value, where in this histogram does it fall?
+    # Returns a float between 0 and 1
+    def approximate_quantile_of_value(v)
+      mutex.synchronize do
+        return 100 if total == 0
+
+        count_examined = 0
+
+        bins.each_with_index do |bin, index|
+          if v <= bin.value
+            break
+          end
+
+          count_examined += bin.count
+        end
+
+        count_examined / total.to_f
+      end
+    end
+
     def mean
       mutex.synchronize do
         if total == 0

data/lib/scout_apm/instruments/percentile_sampler.rb

@@ -0,0 +1,37 @@
+module ScoutApm
+  module Instruments
+    class PercentileSampler
+      attr_reader :logger
+
+      attr_reader :percentiles
+
+      def initialize(logger, percentiles)
+        @logger = logger
+        @percentiles = Array(percentiles)
+      end
+
+      def human_name
+        "Percentiles"
+      end
+
+      # Gets the 95th%ile for the time requested
+      def metrics(time)
+        ms = {}
+        histos = ScoutApm::Agent.instance.request_histograms_by_time[time]
+        histos.each_name do |name|
+          percentiles.each do |percentile|
+            meta = MetricMeta.new("Percentile/#{percentile}/#{name}")
+            stat = MetricStats.new
+            stat.update!(histos.quantile(name, percentile))
+            ms[meta] = stat
+          end
+        end
+
+        # Wipe the histograms we just collected data on
+        ScoutApm::Agent.instance.request_histograms_by_time.delete(time)
+
+        ms
+      end
+    end
+  end
+end

data/lib/scout_apm/instruments/process/process_cpu.rb

@@ -29,6 +29,18 @@ module ScoutApm
           "Process CPU"
         end
 
+        def metrics(_time)
+          result = run
+          if result
+            meta = MetricMeta.new("#{metric_type}/#{metric_name}")
+            stat = MetricStats.new(false)
+            stat.update!(result)
+            { meta => stat }
+          else
+            {}
+          end
+        end
+
         # TODO: Figure out a good default instead of nil
         def run
           res = nil

data/lib/scout_apm/instruments/process/process_memory.rb

@@ -20,6 +20,18 @@ module ScoutApm
           "Process Memory"
         end
 
+        def metrics(_time)
+          result = run
+          if result
+            meta = MetricMeta.new("#{metric_type}/#{metric_name}")
+            stat = MetricStats.new(false)
+            stat.update!(result)
+            { meta => stat }
+          else
+            {}
+          end
+        end
+
         def run
           case RUBY_PLATFORM.downcase
           when /linux/

data/lib/scout_apm/layer_converters/converter_base.rb

@@ -19,10 +19,12 @@ module ScoutApm
       #       render :update
       #     end
       def scope_layer
-        @scope_layer ||= walker.walk do |layer|
-          if layer.type == "Controller"
-            break layer
-          end
+        @scope_layer ||= find_first_layer_of_type("Controller") || find_first_layer_of_type("Job")
+      end
+
+      def find_first_layer_of_type(layer_type)
+        walker.walk do |layer|
+          return layer if layer.type == layer_type
         end
       end
     end

data/lib/scout_apm/layer_converters/slow_job_converter.rb

@@ -4,15 +4,29 @@ module ScoutApm
       def initialize(*)
         @backtraces = []
         super
+
+        # After call to super, so @request is populated
+        @points = if request.job?
+                    ScoutApm::Agent.instance.slow_job_policy.score(request)
+                  else
+                    -1
+                  end
       end
 
-      def call
-        return unless request.job?
+      def name
+        request.unique_name
+      end
+
+      def score
+        @points
+      end
 
-        job_name = [queue_layer.name, job_layer.name]
+      def call
+        return nil unless request.job?
+        return nil unless queue_layer
+        return nil unless job_layer
 
-        slow_enough = ScoutApm::Agent.instance.slow_job_policy.slow?(job_name, root_layer.total_call_time)
-        return unless slow_enough
+        ScoutApm::Agent.instance.slow_job_policy.stored!(request)
 
         SlowJobRecord.new(
           queue_layer.name,
@@ -21,7 +35,8 @@ module ScoutApm
           job_layer.total_call_time,
           job_layer.total_exclusive_time,
           request.context,
-          create_metrics)
+          create_metrics,
+          score)
       end
 
       def queue_layer
@@ -32,12 +47,6 @@ module ScoutApm
         @job_layer ||= find_first_layer_of_type("Job")
       end
 
-      def find_first_layer_of_type(layer_type)
-        walker.walk do |layer|
-          return layer if layer.type == layer_type
-        end
-      end
-
       def create_metrics
         metric_hash = Hash.new
 
@@ -102,7 +111,6 @@ module ScoutApm
       end
 
       def attach_backtraces(metric_hash)
-        ScoutApm::Agent.instance.logger.info("Attaching backtraces to job #{@backtraces}")
         @backtraces.each do |meta_with_backtrace|
           metric_hash.keys.find { |k| k == meta_with_backtrace }.backtrace = meta_with_backtrace.backtrace
         end

data/lib/scout_apm/layer_converters/slow_request_converter.rb

@@ -4,46 +4,52 @@ module ScoutApm
       def initialize(*)
         @backtraces = [] # An Array of MetricMetas that have a backtrace
         super
+
+        # After call to super, so @request is populated
+        @points = if request.web?
+                    ScoutApm::Agent.instance.slow_request_policy.score(request)
+                  else
+                    -1
+                  end
+      end
+
+      def name
+        request.unique_name
+      end
+
+      def score
+        @points
       end
 
+      # Unconditionally attempts to convert this into a SlowTransaction object.
+      # Can return nil if the request didn't have any scope_layer.
       def call
         scope = scope_layer
-        return [nil, {}] unless scope
-
-        policy = ScoutApm::Agent.instance.slow_request_policy.capture_type(root_layer.total_call_time)
-        if policy == ScoutApm::SlowRequestPolicy::CAPTURE_NONE
-          return [nil, {}]
-        end
+        return nil unless scope
 
-        # increment the slow transaction count if this is a slow transaction.
-        meta = MetricMeta.new("SlowTransaction/#{scope.legacy_metric_name}")
-        stat = MetricStats.new
-        stat.update!(1)
+        ScoutApm::Agent.instance.slow_request_policy.stored!(request)
 
         uri = request.annotations[:uri] || ""
 
         ScoutApm::Agent.instance.config.value("ignore_traces").each do |pattern|
           if /#{pattern}/ =~ uri
             ScoutApm::Agent.instance.logger.debug("Skipped recording a trace for #{uri} due to `ignore_traces` pattern: #{pattern}")
-            return [nil, { meta => stat }]
+            return nil
           end
         end
 
-
         metrics = create_metrics
         # Disable stackprof output for now
         stackprof = [] # request.stackprof
 
-        [
-          SlowTransaction.new(uri,
-                              scope.legacy_metric_name,
-                              root_layer.total_call_time,
-                              metrics,
-                              request.context,
-                              root_layer.stop_time,
-                              stackprof),
-          { meta => stat }
-        ]
+        SlowTransaction.new(uri,
+                            scope.legacy_metric_name,
+                            root_layer.total_call_time,
+                            metrics,
+                            request.context,
+                            root_layer.stop_time,
+                            stackprof,
+                            @points)
       end
 
       # Iterates over the TrackedRequest's MetricMetas that have backtraces and attaches each to correct MetricMeta in the Metric Hash.