scout_apm 2.5.1 → 5.3.2

data/lib/scout_apm/config.rb

@@ -29,10 +29,19 @@ require 'scout_apm/environment'
 # report_format    - 'json' or 'marshal'. Marshal is legacy and will be removed.
 # scm_subdirectory - if the app root lives in source management in a subdirectory. E.g. #{SCM_ROOT}/src
 # uri_reporting    - 'path' or 'full_path' default is 'full_path', which reports URL params as well as the path.
+# record_queue_time - true/false to enable recording of queuetime.
 # remote_agent_host - Internal: What host to bind to, and also send messages to for remote. Default: 127.0.0.1.
 # remote_agent_port - What port to bind the remote webserver to
 # start_resque_server_instrument - Used in special situations with certain Resque installs
 # timeline_traces - true/false to enable sending of of the timeline trace format.
+# auto_instruments - true/false whether to install autoinstruments. Only installed if on a supported Ruby version.
+# auto_instruments_ignore  - An array of file names to exclude from autoinstruments (Ex: ['application_controller']).
+# use_prepend              - Whether to apply instrumentation using Module#Prepend instead
+#                            of Module#alias_method (Default: false)
+# alias_method_instruments - If `use_prepend` is true, continue to use Module#alias_method for
+#                            any instruments listed in this array. Default: []
+# prepend_instruments      - If `use_prepend` is false, force using Module#prepend for any
+#                            instruments listed in this array. Default: []
 #
 # Any of these config settings can be set with an environment variable prefixed
 # by SCOUT_ and uppercasing the key: SCOUT_LOG_LEVEL for instance.
@@ -53,6 +62,8 @@ module ScoutApm
         'direct_host',
         'disabled_instruments',
         'enable_background_jobs',
+        'external_service_metric_limit',
+        'external_service_metric_report_limit',
         'host',
         'hostname',
         'ignore',
@@ -67,15 +78,28 @@ module ScoutApm
         'name',
         'profile',
         'proxy',
+        'record_queue_time',
         'remote_agent_host',
         'remote_agent_port',
         'report_format',
         'revision_sha',
         'scm_subdirectory',
         'start_resque_server_instrument',
+        'ssl_cert_file',
         'uri_reporting',
         'instrument_http_url_length',
-        'timeline_traces'
+        'timeline_traces',
+        'auto_instruments',
+        'auto_instruments_ignore',
+        'use_prepend',
+        'alias_method_instruments',
+        'prepend_instruments',
+
+        # Error Service Related Configuration
+        'errors_enabled',
+        'errors_ignored_exceptions',
+        'errors_filtered_params',
+        'errors_host',
     ]
 
     ################################################################################
@@ -166,9 +190,20 @@ module ScoutApm
       'compress_payload' => BooleanCoercion.new,
       'database_metric_limit'  => IntegerCoercion.new,
       'database_metric_report_limit' => IntegerCoercion.new,
+      'external_service_metric_limit'  => IntegerCoercion.new,
+      'external_service_metric_report_limit' => IntegerCoercion.new,
       'instrument_http_url_length' => IntegerCoercion.new,
+      'record_queue_time' => BooleanCoercion.new,
       'start_resque_server_instrument' => BooleanCoercion.new,
-      'timeline_traces' => BooleanCoercion.new
+      'timeline_traces' => BooleanCoercion.new,
+      'auto_instruments' => BooleanCoercion.new,
+      'auto_instruments_ignore' => JsonCoercion.new,
+      'use_prepend' => BooleanCoercion.new,
+      'alias_method_instruments' => JsonCoercion.new,
+      'prepend_instruments' => JsonCoercion.new,
+      'errors_enabled' => BooleanCoercion.new,
+      'errors_ignored_exceptions' => JsonCoercion.new,
+      'errors_filtered_params' => JsonCoercion.new,
     }
 
 
@@ -273,10 +308,23 @@ module ScoutApm
         'remote_agent_port'      => 7721, # picked at random
         'database_metric_limit'  => 5000, # The hard limit on db metrics
         'database_metric_report_limit' => 1000,
+        'external_service_metric_limit'  => 5000, # The hard limit on external service metrics
+        'external_service_metric_report_limit' => 1000,
         'instrument_http_url_length' => 300,
         'start_resque_server_instrument' => true, # still only starts if Resque is detected
         'collect_remote_ip' => true,
-        'timeline_traces' => true
+        'record_queue_time' => true,
+        'timeline_traces' => true,
+        'auto_instruments' => false,
+        'auto_instruments_ignore' => [],
+        'use_prepend' => false,
+        'alias_method_instruments' => [],
+        'prepend_instruments' => [],
+        'ssl_cert_file' => File.join( File.dirname(__FILE__), *%w[.. .. data cacert.pem] ),
+        'errors_enabled' => false,
+        'errors_ignored_exceptions' => %w(ActiveRecord::RecordNotFound ActionController::RoutingError),
+        'errors_filtered_params' => %w(password s3-key),
+        'errors_host' => 'https://errors.scoutapm.com',
       }.freeze
 
       def value(key)
@@ -399,7 +447,7 @@ module ScoutApm
         begin
           raw_file = File.read(@resolved_file_path)
           erb_file = ERB.new(raw_file).result(binding)
-          parsed_yaml = YAML.load(erb_file)
+          parsed_yaml = YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(erb_file) : YAML.load(erb_file)
           file_settings = parsed_yaml[app_environment]
 
           if file_settings.is_a? Hash
@@ -410,8 +458,8 @@ module ScoutApm
             logger.info("Couldn't find configuration in #{@resolved_file_path} for environment: #{app_environment}. Configuration in ENV will still be applied.")
             @file_loaded = false
           end
-        rescue Exception => e # Explicit `Exception` handling to catch SyntaxError and anything else that ERB or YAML may throw
-          logger.info("Failed loading configuration file (#{@resolved_file_path}): #{e.message}. ScoutAPM will continue starting with configuration from ENV and defaults")
+        rescue ScoutApm::AllExceptionsExceptOnesWeMustNotRescue => e # Everything except the most important exceptions we should never interfere with
+          logger.info("Failed loading configuration file (#{@resolved_file_path}): ScoutAPM will continue starting with configuration from ENV and defaults. Exception was #{e.class}: #{e.message}#{e.backtrace.map { |bt| "\n  #{bt}" }.join('')}")
           @file_loaded = false
         end
       end

data/lib/scout_apm/detailed_trace.rb

@@ -98,7 +98,7 @@ class DetailedTrace
         }
       },
       :tags => tags.as_json,
-      :spans => spans.as_json,
+      :spans => spans.map{|span| span.as_json},
     }
   end
 
@@ -200,8 +200,9 @@ class DetailedTraceTags
     @tags = hash
   end
 
+  # @tags is already a hash, so no conversion needed
   def as_json(*)
-    @tags.as_json
+    @tags
   end
 end
 

data/lib/scout_apm/environment.rb

@@ -29,6 +29,8 @@ module ScoutApm
       ScoutApm::BackgroundJobIntegrations::Shoryuken.new,
       ScoutApm::BackgroundJobIntegrations::Sneakers.new,
       ScoutApm::BackgroundJobIntegrations::DelayedJob.new,
+      ScoutApm::BackgroundJobIntegrations::Que.new,
+      ScoutApm::BackgroundJobIntegrations::Faktory.new,
     ]
 
     FRAMEWORK_INTEGRATIONS = [
@@ -181,9 +183,24 @@ module ScoutApm
       @ruby_2 = defined?(RUBY_VERSION) && RUBY_VERSION.match(/^2/)
     end
 
+    def ruby_3?
+      return @ruby_3 if defined?(@ruby_3)
+      @ruby_3 = defined?(RUBY_VERSION) && RUBY_VERSION.match(/^3/)
+    end
+
+    def ruby_minor
+      return @ruby_minor if defined?(@ruby_minor)
+      @ruby_minor = defined?(RUBY_VERSION) && RUBY_VERSION.split(".")[1].to_i
+    end
+
     # Returns true if this Ruby version supports Module#prepend.
     def supports_module_prepend?
-      ruby_2?
+      ruby_2? || ruby_3?
+    end
+
+    # Returns true if this Ruby version makes positional and keyword arguments incompatible
+    def supports_kwarg_delegation?
+      ruby_3? || (ruby_2? && ruby_minor >= 7)
     end
 
     # Returns a string representation of the OS (ex: darwin, linux)

data/lib/scout_apm/error.rb

@@ -0,0 +1,27 @@
+# Public API for the Scout Error Monitoring service
+#
+# See-Also ScoutApm::Transaction and ScoutApm::Tracing for APM related APIs
+module ScoutApm
+  module Error
+    # Capture an exception, optionally with an environment hash. This may be a
+    # Rack environment, but is not required.
+    def self.capture(exception, env={})
+      context = ScoutApm::Agent.instance.context
+
+      # Skip if error monitoring isn't enabled at all
+      if ! context.config.value("errors_enabled")
+        return false
+      end
+
+      # Skip if this one error is ignored
+      if context.ignored_exceptions.ignored?(exception)
+        return false
+      end
+
+      # Capture the error for further processing and shipping
+      context.error_buffer.capture(exception, env)
+
+      return true
+    end
+  end
+end

data/lib/scout_apm/error_service/error_buffer.rb

@@ -0,0 +1,39 @@
+# Holds onto exceptions, and moves them forward to shipping when appropriate
+module ScoutApm
+  module ErrorService
+    class ErrorBuffer
+      include Enumerable
+
+      attr_reader :agent_context
+
+      def initialize(agent_context)
+        @agent_context = agent_context
+        @error_records = []
+        @mutex = Monitor.new
+      end
+
+      def capture(exception, env)
+        context = ScoutApm::Context.current
+
+        @mutex.synchronize {
+          @error_records << ErrorRecord.new(agent_context, exception, env, context)
+        }
+      end
+
+      def get_and_reset_error_records
+        @mutex.synchronize {
+          ret = @error_records
+          @error_records = []
+          ret
+        }
+      end
+
+      # Enables enumerable - for count and each and similar methods
+      def each
+        @error_records.each do |error_record|
+          yield error_record
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/error_service/error_record.rb

@@ -0,0 +1,211 @@
+module ScoutApm
+  module ErrorService
+    # Converts the raw error data captured into the captured data, and holds it
+    # until it's ready to be reported.
+    class ErrorRecord
+      attr_reader :exception_class
+      attr_reader :message
+      attr_reader :request_uri
+      attr_reader :request_params
+      attr_reader :request_session
+      attr_reader :environment
+      attr_reader :trace
+      attr_reader :request_components
+      attr_reader :context
+
+      def initialize(agent_context, exception, env, context=nil)
+        @agent_context = agent_context
+
+        @context = if context
+          context.to_hash
+        else
+          {}
+        end
+
+        @exception_class = LengthLimit.new(exception.class.name).to_s
+        @message = LengthLimit.new(exception.message, 100).to_s
+        @request_uri = LengthLimit.new(rack_request_url(env), 200).to_s
+        @request_params = clean_params(env["action_dispatch.request.parameters"])
+        @request_session = clean_params(session_data(env))
+        @environment = clean_params(strip_env(env))
+        @trace = clean_backtrace(exception.backtrace)
+        @request_components = components(env)
+      end
+
+      # TODO: This is rails specific
+      def components(env)
+        components = {}
+        unless env["action_dispatch.request.parameters"].nil?
+          components[:controller] = env["action_dispatch.request.parameters"][:controller] || nil
+          components[:action] = env["action_dispatch.request.parameters"][:action] || nil
+          components[:module] = env["action_dispatch.request.parameters"][:module] || nil
+        end
+
+        # For background workers like sidekiq
+        # TODO: extract data creation for background jobs
+        components[:controller] ||= env[:custom_controller]
+
+        components
+      end
+
+      # TODO: Can I use the same thing we use in traces?
+      def rack_request_url(env)
+        protocol = rack_scheme(env)
+        protocol = protocol.nil? ? "" : "#{protocol}://"
+
+        host = env["SERVER_NAME"] || ""
+        path = env["REQUEST_URI"] || ""
+        port = env["SERVER_PORT"] || "80"
+        port = ["80", "443"].include?(port.to_s) ? "" : ":#{port}"
+
+        protocol.to_s + host.to_s + port.to_s + path.to_s
+      end
+
+      def rack_scheme(env)
+        if env["HTTPS"] == "on"
+          "https"
+        elsif env["HTTP_X_FORWARDED_PROTO"]
+          env["HTTP_X_FORWARDED_PROTO"].split(",")[0]
+        else
+          env["rack.url_scheme"]
+        end
+      end
+
+      # TODO: This name is too vague
+      def clean_params(params)
+        return if params.nil?
+
+        normalized = normalize_data(params)
+        filter_params(normalized)
+      end
+
+      # TODO: When was backtrace_cleaner introduced?
+      def clean_backtrace(backtrace)
+        if defined?(Rails) && Rails.respond_to?(:backtrace_cleaner)
+          Rails.backtrace_cleaner.send(:filter, backtrace)
+        else
+          backtrace
+        end
+      end
+
+      # Deletes params from env
+      #
+      # These are not configurable, and will leak PII info up to Scout if
+      # allowed through. Things like specific parameters can be exposed with
+      # the ScoutApm::Context interface.
+      KEYS_TO_REMOVE = [
+        "rack.request.form_hash",
+        "rack.request.form_vars",
+        "async.callback",
+
+        # Security related items
+        "action_dispatch.secret_key_base",
+        "action_dispatch.http_auth_salt",
+        "action_dispatch.signed_cookie_salt",
+        "action_dispatch.encrypted_cookie_salt",
+        "action_dispatch.encrypted_signed_cookie_salt",
+        "action_dispatch.authenticated_encrypted_cookie_salt",
+
+        # Raw data from the URL & parameters. Would bypass our normal params filtering
+        "QUERY_STRING",
+        "REQUEST_URI",
+        "REQUEST_PATH",
+        "ORIGINAL_FULLPATH",
+        "action_dispatch.request.query_parameters",
+        "action_dispatch.request.parameters",
+        "rack.request.query_string",
+        "rack.request.query_hash",
+      ]
+      def strip_env(env)
+        env.reject { |k, v| KEYS_TO_REMOVE.include?(k) }
+      end
+
+      def session_data(env)
+        session = env["action_dispatch.request.session"]
+        return if session.nil?
+
+        if session.respond_to?(:to_hash)
+          session.to_hash
+        else
+          session.data
+        end
+      end
+
+      # TODO: Rename and make this clearer. I think it maps over the whole tree of a hash, and to_s each leaf node?
+      def normalize_data(hash)
+        new_hash = {}
+
+        hash.each do |key, value|
+          if value.respond_to?(:to_hash)
+            begin
+              new_hash[key] = normalize_data(value.to_hash)
+            rescue
+              new_hash[key] = LengthLimit.new(value.to_s).to_s
+            end
+          else
+            new_hash[key] = LengthLimit.new(value.to_s).to_s
+          end
+        end
+
+        new_hash
+      end
+
+      ###################
+      # Filtering Params
+      ###################
+
+      # Replaces parameter values with a string / set in config file
+      def filter_params(params)
+        return params unless filtered_params_config
+
+        params.each do |k, v|
+          if filter_key?(k)
+            params[k] = "[FILTERED]"
+          elsif v.respond_to?(:to_hash)
+            filter_params(params[k])
+          end
+        end
+
+        params
+      end
+
+      # Check, if a key should be filtered
+      def filter_key?(key)
+        params_to_filter.any? do |filter|
+          key.to_s == filter.to_s # key.to_s.include?(filter.to_s)
+        end
+      end
+
+      def params_to_filter
+        @params_to_filter ||= filtered_params_config + rails_filtered_params
+      end
+
+      # Accessor for the filtered params config value. Will be removed as we refactor and clean up this code.
+      # TODO: Flip this over to use a new class like filtered exceptions?
+      def filtered_params_config
+        @agent_context.config.value("errors_filtered_params")
+      end
+
+      def rails_filtered_params
+        return [] unless defined?(Rails)
+        Rails.configuration.filter_parameters
+      rescue 
+        []
+      end
+
+      class LengthLimit
+        attr_reader :text
+        attr_reader :char_limit
+
+        def initialize(text, char_limit=100)
+          @text = text
+          @char_limit = char_limit
+        end
+
+        def to_s
+          text[0..char_limit]
+        end
+      end
+    end
+  end
+end

data/lib/scout_apm/error_service/ignored_exceptions.rb

@@ -0,0 +1,66 @@
+# Encapsulates the management and checking of ignored exceptions. Allows using
+# string matches on the class name, or arbitrary matching with a callback
+module ScoutApm
+  module ErrorService
+    class IgnoredExceptions
+      attr_reader :ignored_exceptions
+      attr_reader :blocks
+
+      def initialize(context, from_config)
+        @context = context
+        @ignored_exceptions = Array(from_config).map{ |e| normalize_as_klass(e) }
+        @blocks = []
+      end
+
+      # Add a single ignored exception by class name
+      def add(klass_or_str)
+        @ignored_exceptions << normalize_as_klass(klass_or_str)
+      end
+
+      # Add a callback block that will be called on every error. If it returns
+      # Signature of blocks:  ->(exception object): truthy or falsy value
+      def add_callback(&block)
+        @blocks << block
+      end
+
+      def ignored?(exception_object)
+        klass = normalize_as_klass(exception_object)
+
+        # Check if we ignored this error by name (typical way to ignore)
+        if ignored_exceptions.any? { |ignored| klass.ancestors.include?(ignored) }
+          return true
+        end
+
+        # For each block, see if it says we should ignore this error
+        blocks.each do |b|
+          if b.call(exception_object)
+            return true
+          end
+        end
+
+        false
+      end
+
+      private
+
+      def normalize_as_klass(klass_or_str)
+        if Module === klass_or_str
+          return klass_or_str
+        end
+
+        if klass_or_str.is_a?(Exception)
+          return klass_or_str.class
+        end
+
+        if String === klass_or_str
+          maybe = ScoutApm::Utils::KlassHelper.lookup(klass_or_str)
+          if Module === maybe
+            return maybe
+          end
+        end
+
+        klass_or_str
+      end
+    end
+  end
+end

data/lib/scout_apm/error_service/middleware.rb

@@ -0,0 +1,32 @@
+module ScoutApm
+  module ErrorService
+    class Middleware
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        begin
+          response = @app.call(env)
+        rescue Exception => exception
+          context = ScoutApm::Agent.instance.context
+
+          context.logger.debug "[Scout Error Service] Caught Exception: #{exception.class.name}"
+
+          # Bail out early, and reraise if the error is not interesting.
+          if context.ignored_exceptions.ignored?(exception)
+            raise
+          end
+
+          # Capture the error for further processing and shipping
+          context.error_buffer.capture(exception, env)
+
+          # Finally re-raise
+          raise
+        end
+
+        response
+      end
+    end
+  end
+end

data/lib/scout_apm/error_service/notifier.rb

@@ -0,0 +1,33 @@
+module ScoutApm
+  module ErrorService
+    class Notifier
+      attr_reader :context
+      attr_reader :reporter
+
+      def initialize(context)
+        @context = context
+        @reporter = ScoutApm::Reporter.new(context, :errors)
+      end
+
+      def ship
+        error_records = context.error_buffer.get_and_reset_error_records
+        if error_records.any?
+          payload = ScoutApm::ErrorService::Payload.new(context, error_records)
+          reporter.report(
+            payload.serialize,
+            default_headers.merge("X-Error-Count" => error_records.length.to_s)
+          )
+        end
+      end
+
+      private
+
+      def default_headers
+        {
+          "Content-Type" => "application/json",
+          "Accept" => "application/json"
+        }
+      end
+    end
+  end
+end

data/lib/scout_apm/error_service/payload.rb

@@ -0,0 +1,47 @@
+module ScoutApm
+  module ErrorService
+    class Payload
+      attr_reader :context
+      attr_reader :errors
+
+      def initialize(context, errors)
+        @context = context
+        @errors = errors
+      end
+
+      # TODO: Don't use to_json since it isn't supported in Ruby 1.8.7
+      def serialize
+        payload = as_json.to_json
+        context.logger.debug(payload)
+        payload
+      end
+
+      def as_json
+        serialized_errors = errors.map do |error_record|
+          serialize_error_record(error_record)
+        end
+
+        {
+          :notifier => "scout_apm_ruby",
+          :environment => context.environment.env,
+          :root => context.environment.root,
+          :problems => serialized_errors,
+        }
+      end
+
+      def serialize_error_record(error_record)
+        {
+          :exception_class => error_record.exception_class,
+          :message => error_record.message,
+          :request_uri => error_record.request_uri,
+          :request_params => error_record.request_params,
+          :request_session => error_record.request_session,
+          :environment => error_record.environment,
+          :trace => error_record.trace,
+          :request_components => error_record.request_components,
+          :context => error_record.context,
+        }
+      end
+    end
+  end
+end

data/lib/scout_apm/error_service/periodic_work.rb

@@ -0,0 +1,17 @@
+module ScoutApm
+  module ErrorService
+    class PeriodicWork
+      attr_reader :context
+
+      def initialize(context)
+        @context = context
+        @notifier = ScoutApm::ErrorService::Notifier.new(context)
+      end
+
+      # Expected to be called many times over the life of the agent
+      def run
+        @notifier.ship
+      end
+    end
+  end
+end

data/lib/scout_apm/error_service/railtie.rb

@@ -0,0 +1,11 @@
+module ScoutApm
+  module ErrorService
+    class Railtie < Rails::Railtie
+      initializer "scoutapm_error_service.middleware" do |app|
+        next if ScoutApm::Agent.instance.config.value("error_service")
+
+        app.config.middleware.insert_after ActionDispatch::DebugExceptions, ScoutApm::ErrorService::Rack
+      end
+    end
+  end
+end