scout_apm 2.1.32 → 2.2.0.pre0

data/lib/scout_apm/attribute_arranger.rb

@@ -5,6 +5,8 @@ module ScoutApm
     def self.call(subject, attributes_list)
       attributes_list.inject({}) do |attribute_hash, attribute|
         case attribute
+        when :traces
+          attribute_hash[attribute] = subject.traces.to_a
         when Array
           attribute_hash[attribute[0]] = subject.send(attribute[1])
         when :bucket

data/lib/scout_apm/background_job_integrations/delayed_job.rb

@@ -1,9 +1,6 @@
 module ScoutApm
   module BackgroundJobIntegrations
     class DelayedJob
-      ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::DelayedJobAdapter::JobWrapper'.freeze
-      DJ_PERFORMABLE_METHOD = 'Delayed::PerformableMethod'.freeze
-
       attr_reader :logger
 
       def name
@@ -11,79 +8,12 @@ module ScoutApm
       end
 
       def present?
-        defined?(::Delayed::Job)
+        defined?(::Delayed::Job) && (File.basename($0) =~ /\Adelayed_job/)
       end
 
       def forking?
         false
       end
-
-      def install
-        plugin = Class.new(Delayed::Plugin) do
-          require 'delayed_job'
-
-          callbacks do |lifecycle|
-            lifecycle.around(:invoke_job) do |job, *args, &block|
-              ScoutApm::Agent.instance.start_background_worker unless ScoutApm::Agent.instance.background_worker_running?
-
-              name = begin
-                       case job.payload_object.class.to_s
-
-                       # ActiveJob's class wraps the actual job class
-                       when ACTIVE_JOB_KLASS
-                         job.payload_object.job_data["job_class"]
-
-                       # An adhoc job, called like `@user.delay.fib(10)`.
-                       # returns a string like "User#fib"
-                       when DJ_PERFORMABLE_METHOD
-                         job.name
-
-                       # A "real" job called like `Delayed::Job.enqueue(MyJob.new)`
-                       # returns "MyJob"
-                       else
-                         job.payload_object.class.to_s
-                       end
-                     rescue
-                       # Fall back to whatever DJ thinks the name is.
-                       job.name
-                     end
-
-              queue = job.queue || "default"
-
-              req = ScoutApm::RequestManager.lookup
-              req.job!
-
-              begin
-                latency = Time.now - job.created_at
-                req.annotate_request(:queue_latency => latency)
-              rescue
-              end
-
-              queue_layer = ScoutApm::Layer.new('Queue', queue)
-              job_layer = ScoutApm::Layer.new('Job', name)
-
-              begin
-                req.start_layer(queue_layer)
-                started_queue = true
-                req.start_layer(job_layer)
-                started_job = true
-
-                # Call the job itself.
-                block.call(job, *args)
-              rescue
-                req.error!
-                raise
-              ensure
-                req.stop_layer if started_job
-                req.stop_layer if started_queue
-              end
-            end
-          end
-        end
-
-        Delayed::Worker.plugins << plugin # ScoutApm::BackgroundJobIntegrations::DelayedJobPlugin
-      end
     end
   end
 end
-

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -8,7 +8,7 @@ module ScoutApm
       end
 
       def present?
-        defined?(::Sidekiq) && File.basename($PROGRAM_NAME).start_with?('sidekiq')
+        defined?(::Sidekiq) && (File.basename($0) =~ /\Asidekiq/)
       end
 
       def forking?
@@ -16,33 +16,22 @@ module ScoutApm
       end
 
       def install
-        install_tracer
-        add_middleware
-        install_processor
-      end
-
-      def install_tracer
         # ScoutApm::Tracer is not available when this class is defined
         SidekiqMiddleware.class_eval do
           include ScoutApm::Tracer
         end
-      end
 
-      def add_middleware
         ::Sidekiq.configure_server do |config|
           config.server_middleware do |chain|
             chain.add SidekiqMiddleware
           end
         end
-      end
 
-      def install_processor
         require 'sidekiq/processor' # sidekiq v4 has not loaded this file by this point
 
         ::Sidekiq::Processor.class_eval do
           def initialize_with_scout(boss)
-            agent = ::ScoutApm::Agent.instance
-            agent.start_background_worker
+            ::ScoutApm::Agent.instance.start_background_worker unless ::ScoutApm::Agent.instance.background_worker_running?
             initialize_without_scout(boss)
           end
 
@@ -52,69 +41,41 @@ module ScoutApm
       end
     end
 
-    # We insert this middleware into the Sidekiq stack, to capture each job,
-    # and time them.
     class SidekiqMiddleware
-      def call(_worker, msg, queue)
+      def call(worker, msg, queue)
+        job_class = msg["class"] # TODO: Validate this across different versions of Sidekiq
+        if job_class == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" && msg.has_key?("wrapped")
+          job_class = msg["wrapped"]
+        end
+
+        latency = (Time.now.to_f - (msg['enqueued_at'] || msg['created_at']))
+
         req = ScoutApm::RequestManager.lookup
         req.job!
-        req.annotate_request(:queue_latency => latency(msg))
+        req.annotate_request(:queue_latency => latency)
 
-        begin
-          req.start_layer(ScoutApm::Layer.new('Queue', queue))
-          started_queue = true
-          req.start_layer(ScoutApm::Layer.new('Job', job_class(msg)))
-          started_job = true
+        queue_layer = ScoutApm::Layer.new("Queue", queue)
+        job_layer = ScoutApm::Layer.new("Job", job_class)
+
+        #if ScoutApm::Agent.instance.config.value('profile')
+          # Capture ScoutProf if we can
+          #req.enable_profiled_thread!
+          #job_layer.set_root_class(job_class)
+          #job_layer.traced!
+        #end
 
+        req.start_layer(queue_layer)
+        req.start_layer(job_layer)
+
+        begin
           yield
         rescue
           req.error!
           raise
-        ensure
-          req.stop_layer if started_job
-          req.stop_layer if started_queue
-        end
-      end
-
-      UNKNOWN_CLASS_PLACEHOLDER = 'UnknownJob'.freeze
-      ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper'.freeze
-      DELAYED_WRAPPER_KLASS = 'Sidekiq::Extensions::DelayedClass'.freeze
-
-
-      def job_class(msg)
-        job_class = msg.fetch('class', UNKNOWN_CLASS_PLACEHOLDER)
-
-        if job_class == ACTIVE_JOB_KLASS && msg.key?('wrapped')
-          begin
-            job_class = msg['wrapped']
-          rescue
-            ACTIVE_JOB_KLASS
-          end
-        elsif job_class == DELAYED_WRAPPER_KLASS
-          begin
-            yml = msg['args'].first
-            deserialized_args = YAML.load(yml)
-            klass, method, *rest = deserialized_args
-            job_class = [klass,method].map(&:to_s).join(".")
-          rescue
-            DELAYED_WRAPPER_KLASS
-          end
-        end
-
-        job_class
-      rescue
-        UNKNOWN_CLASS_PLACEHOLDER
-      end
-
-      def latency(msg, time = Time.now.to_f)
-        created_at = msg['enqueued_at'] || msg['created_at']
-        if created_at
-          (time - created_at)
-        else
-          0
         end
-      rescue
-        0
+      ensure
+        req.stop_layer # Job
+        req.stop_layer # Queue
       end
     end
   end

data/lib/scout_apm/background_worker.rb

@@ -11,10 +11,6 @@ module ScoutApm
       @keep_running = true
     end
 
-    def running?
-      @keep_running
-    end
-
     def stop
       ScoutApm::Agent.instance.logger.debug "Background Worker: stop requested"
       @keep_running = false
@@ -29,13 +25,19 @@ module ScoutApm
     def start(&block)
       @task = block
 
-      ScoutApm::Agent.instance.logger.debug "Background Worker: Starting Background Worker, running every #{period} seconds"
+      begin
+        ScoutApm::Agent.instance.logger.debug "Background Worker: Starting Background Worker, running every #{period} seconds"
 
-      # The first run should be 1 period of time from now
-      next_time = Time.now + period
+        # The first run should be 1 period of time from now
+        next_time = Time.now + period
+
+        loop do
+          # Bail out if @keep_running is false
+          unless @keep_running
+            ScoutApm::Agent.instance.logger.debug "Background Worker: breaking from loop"
+            break
+          end
 
-      loop do
-        begin
           now = Time.now
 
           # Sleep the correct amount of time to reach next_time
@@ -45,23 +47,17 @@ module ScoutApm
             now = Time.now
           end
 
-          # Bail out if @keep_running is false
-          unless @keep_running
-            ScoutApm::Agent.instance.logger.debug "Background Worker: breaking from loop"
-            break
-          end
-
           @task.call
 
           # Adjust the next time to run forward by @periods until it is in the future
           while next_time <= now
             next_time += period
           end
-        rescue
-          ScoutApm::Agent.instance.logger.debug "Background Worker Exception!"
-          ScoutApm::Agent.instance.logger.debug $!.message
-          ScoutApm::Agent.instance.logger.debug $!.backtrace
         end
+      rescue
+        ScoutApm::Agent.instance.logger.debug "Background Worker Exception!"
+        ScoutApm::Agent.instance.logger.debug $!.message
+        ScoutApm::Agent.instance.logger.debug $!.backtrace
       end
     end
   end

data/lib/scout_apm/capacity.rb

@@ -0,0 +1,57 @@
+# Encapsulates logic for determining capacity utilization of the Ruby processes.
+module ScoutApm
+  class Capacity
+    attr_reader :processing_start_time, :accumulated_time, :transaction_entry_time
+
+    def initialize
+      @processing_start_time = Time.now
+      @lock ||= Mutex.new # the transaction_entry_time could be modified while processing a request or when #process is called.
+      @accumulated_time = 0.0
+    end
+
+    # Called when a transaction is traced.
+    def start_transaction!
+      @lock.synchronize do
+        @transaction_entry_time = Time.now
+      end
+    end
+
+    # Called when a transaction completes to record its time used.
+    def finish_transaction!
+      @lock.synchronize do
+        if transaction_entry_time
+          @accumulated_time += (Time.now - transaction_entry_time).to_f
+        else
+          ScoutApm::Agent.instance.logger.warn "No transaction entry time. Not recording capacity metrics for transaction."
+        end
+        @transaction_entry_time = nil
+      end
+    end
+
+    # Ran when sending metrics to server. Reports capacity usage metrics.
+    def process
+      process_time = Time.now
+      ScoutApm::Agent.instance.logger.debug "Processing capacity usage for [#{@processing_start_time}] to [#{process_time}]. Time Spent: #{@accumulated_time}."
+      @lock.synchronize do
+        time_spent = @accumulated_time
+        @accumulated_time = 0.0
+        # If a transaction is still running, capture its running time up to now and
+        # reset the +transaction_entry_time+ to now.
+        if @transaction_entry_time
+          time_spent += (process_time - @transaction_entry_time).to_f
+          ScoutApm::Agent.instance.logger.debug "A transaction is running while calculating capacity. Start time: [#{transaction_entry_time}]. Will update the entry time to [#{process_time}]."
+          @transaction_entry_time = process_time # prevent from over-counting capacity usage. update the transaction start time to now.
+        end
+        time_spent = 0.0 if time_spent < 0.0
+
+        window = (process_time - processing_start_time).to_f # time period we are evaulating capacity usage.
+        window = 1.0 if window <= 0.0 # prevent divide-by-zero if clock adjusted.
+        capacity = time_spent / window
+        ScoutApm::Agent.instance.logger.debug "Instance/Capacity: #{capacity}"
+        ScoutApm::Agent.instance.store.track_one!("Instance", "Capacity", capacity)
+
+        @processing_start_time = process_time
+      end
+    end
+  end
+end

data/lib/scout_apm/config.rb

@@ -5,63 +5,30 @@ require 'scout_apm/environment'
 
 # Valid Config Options:
 #
-# This list is complete, but some are old and unused, or for developers of
-# scout_apm itself. See the documentation at http://help.apm.scoutapp.com for
-# customer-focused documentation.
-#
 # application_root - override the detected directory of the application
-# compress_payload - true/false to enable gzipping of payload
 # data_file        - override the default temporary storage location. Must be a location in a writable directory
-# dev_trace        - true or false. Enables always-on tracing in development environmen only
+# host             - override the default hostname detection. Default varies by environment - either system hostname, or PAAS hostname
 # direct_host      - override the default "direct" host. The direct_host bypasses the ingestion pipeline and goes directly to the webserver, and is primarily used for features under development.
-# enable_background_jobs - true or false
-# host             - configuration used in development
-# hostname         - override the default hostname detection. Default varies by environment - either system hostname, or PAAS hostname
 # key              - the account key with Scout APM. Found in Settings in the Web UI
 # log_file_path    - either a directory or "STDOUT".
 # log_level        - DEBUG / INFO / WARN as usual
 # monitor          - true or false.  False prevents any instrumentation from starting
 # name             - override the name reported to APM. This is the name that shows in the Web UI
-# profile          - turn on/off scoutprof (only applicable in Gem versions including scoutprof)
-# proxy            - an http proxy
-# report_format    - 'json' or 'marshal'. Marshal is legacy and will be removed.
-# scm_subdirectory - if the app root lives in source management in a subdirectory. E.g. #{SCM_ROOT}/src
 # uri_reporting    - 'path' or 'full_path' default is 'full_path', which reports URL params as well as the path.
-# remote_agent_host - Internal: What host to bind to, and also send messages to for remote. Default: 127.0.0.1.
-# remote_agent_port - What port to bind the remote webserver to
+# report_format    - 'json' or 'marshal'. Marshal is legacy and will be removed.
+# dev_trace        - true or false. Enables always-on tracing in development environmen only
+# enable_background_jobs - true or false
 #
 # Any of these config settings can be set with an environment variable prefixed
 # by SCOUT_ and uppercasing the key: SCOUT_LOG_LEVEL for instance.
 
+
+# Config - Made up of config overlay
+# Default -> File -> Environment Var
+# QUESTION: How to embed arrays or hashes into ENV?
+
 module ScoutApm
   class Config
-    KNOWN_CONFIG_OPTIONS = [
-        'application_root',
-        'async_recording',
-        'compress_payload',
-        'config_file',
-        'data_file',
-        'detailed_middleware',
-        'dev_trace',
-        'direct_host',
-        'disabled_instruments',
-        'enable_background_jobs',
-        'host',
-        'hostname',
-        'ignore',
-        'key',
-        'log_file_path',
-        'log_level',
-        'monitor',
-        'name',
-        'profile',
-        'proxy',
-        'remote_agent_host',
-        'remote_agent_port',
-        'report_format',
-        'scm_subdirectory',
-        'uri_reporting',
-    ]
 
     ################################################################################
     # Coersions
@@ -134,12 +101,10 @@ module ScoutApm
 
 
     SETTING_COERCIONS = {
-      "async_recording"        => BooleanCoercion.new,
-      "detailed_middleware"    => BooleanCoercion.new,
-      "dev_trace"              => BooleanCoercion.new,
+      "monitor"                => BooleanCoercion.new,
       "enable_background_jobs" => BooleanCoercion.new,
+      "dev_trace"              => BooleanCoercion.new,
       "ignore"                 => JsonCoercion.new,
-      "monitor"                => BooleanCoercion.new,
     }
 
 
@@ -164,7 +129,7 @@ module ScoutApm
     def self.with_file(file_path=nil, config={})
       overlays = [
         ConfigEnvironment.new,
-        ConfigFile.new(file_path, config),
+        ConfigFile.new(file_path, config[:file]),
         ConfigDefaults.new,
         ConfigNull.new,
       ]
@@ -172,61 +137,28 @@ module ScoutApm
     end
 
     def initialize(overlays)
-      @overlays = Array(overlays)
-    end
-
-    # For a given key, what is the first overlay says that it can handle it?
-    def overlay_for_key(key)
-      @overlays.detect{ |overlay| overlay.has_key?(key) }
+      @overlays = overlays
     end
 
     def value(key)
-      if ! KNOWN_CONFIG_OPTIONS.include?(key)
-        ScoutApm::Agent.instance.logger.debug("Requested looking up a unknown configuration key: #{key} (not a problem. Evaluate and add to config.rb)")
-      end
-
-      o = overlay_for_key(key)
-      raw_value = if o
-                    o.value(key)
-                  else
-                    # No overlay said it could handle this key, bail out with nil.
-                    nil
-                  end
+      raw_value = @overlays.detect{ |overlay| overlay.has_key?(key) }.value(key)
 
-      coercion = SETTING_COERCIONS.fetch(key, NullCoercion.new)
+      coercion = SETTING_COERCIONS[key] || NullCoercion.new
       coercion.coerce(raw_value)
     end
 
-    # Did we load anything for configuration?
-    def any_keys_found?
-      @overlays.any? { |overlay| overlay.any_keys_found? }
-    end
-
-    def log_settings
-      messages = KNOWN_CONFIG_OPTIONS.inject([]) do |memo, key|
-        o = overlay_for_key(key)
-        memo << "#{o.name} - #{key}: #{value(key).inspect}"
-      end
-      ScoutApm::Agent.instance.logger.debug("Resolved Setting Values:\n" + messages.join("\n"))
-    end
-
     class ConfigDefaults
       DEFAULTS = {
-        'compress_payload'       => true,
-        'detailed_middleware'    => false,
-        'dev_trace'              => false,
+        'host'                   => 'https://checkin.scoutapp.com',
         'direct_host'            => 'https://apm.scoutapp.com',
+        'log_level'              => 'info',
+        'uri_reporting'          => 'full_path',
+        'report_format'          => 'json',
         'disabled_instruments'   => [],
         'enable_background_jobs' => true,
-        'host'                   => 'https://checkin.scoutapp.com',
         'ignore'                 => [],
-        'log_level'              => 'info',
-        'profile'                => true, # for scoutprof
-        'report_format'          => 'json',
-        'scm_subdirectory'       => '',
-        'uri_reporting'          => 'full_path',
-        'remote_agent_host'      => '127.0.0.1',
-        'remote_agent_port'      => 7721, # picked at random
+        'dev_trace' => false, # false for now so code can live in main branch
+        'profile' => true # for scoutprof
       }.freeze
 
       def value(key)
@@ -236,15 +168,6 @@ module ScoutApm
       def has_key?(key)
         DEFAULTS.has_key?(key)
       end
-
-      # Defaults are here, but not counted as user specified.
-      def any_keys_found?
-        false
-      end
-
-      def name
-        "defaults"
-      end
     end
 
 
@@ -259,14 +182,6 @@ module ScoutApm
       def has_key?(*)
         true
       end
-
-      def any_keys_found?
-        false
-      end
-
-      def name
-        "no-config"
-      end
     end
 
     class ConfigEnvironment
@@ -282,16 +197,6 @@ module ScoutApm
       def key_to_env_key(key)
         'SCOUT_' + key.upcase
       end
-
-      def any_keys_found?
-        KNOWN_CONFIG_OPTIONS.any? { |option|
-          ENV.has_key?(key_to_env_key(option))
-        }
-      end
-
-      def name
-        "environment"
-      end
     end
 
     # Attempts to load a configuration file, and parse it as YAML. If the file
@@ -317,16 +222,6 @@ module ScoutApm
         @settings.has_key?(key)
       end
 
-      def any_keys_found?
-        KNOWN_CONFIG_OPTIONS.any? { |option|
-          @settings.has_key?(option)
-        }
-      end
-
-      def name
-        "config-file"
-      end
-
       private
 
       def load_file(file)
@@ -347,18 +242,17 @@ module ScoutApm
           raw_file = File.read(@resolved_file_path)
           erb_file = ERB.new(raw_file).result(binding)
           parsed_yaml = YAML.load(erb_file)
-          file_settings = parsed_yaml[app_environment]
+          @settings = parsed_yaml[app_environment]
 
-          if file_settings.is_a? Hash
-            logger.debug("Loaded Configuration: #{@resolved_file_path}. Using environment: #{app_environment}")
-            @settings = file_settings
-            @file_loaded = true
-          else
-            logger.info("Couldn't find configuration in #{@resolved_file_path} for environment: #{app_environment}. Configuration in ENV will still be applied.")
-            @file_loaded = false
+          if !@settings.is_a? Hash
+            raise ("Missing environment key for: #{app_environment}. This can happen if the key is missing, or with a malformed configuration file," +
+                   " check that there is a top level #{app_environment} key.")
           end
+
+          logger.debug("Loaded Configuration: #{@resolved_file_path}. Using environment: #{app_environment}")
+          @file_loaded = true
         rescue Exception => e # Explicit `Exception` handling to catch SyntaxError and anything else that ERB or YAML may throw
-          logger.info("Failed loading configuration file (#{@resolved_file_path}): #{e.message}. ScoutAPM will continue starting with configuration from ENV and defaults")
+          logger.debug("Failed loading configuration file: #{e.message}. ScoutAPM will continue starting with configuration from ENV and defaults")
           @file_loaded = false
         end
       end