scout_apm 5.7.1 → 6.0.0

data/lib/scout_apm/auto_instrument/rails.rb

@@ -1,6 +1,10 @@
 
 require 'scout_apm/auto_instrument/layer'
-require 'scout_apm/auto_instrument/parser'
+if defined?(Prism)
+  require 'scout_apm/auto_instrument/prism'
+else
+  require 'scout_apm/auto_instrument/parser'
+end
 
 module ScoutApm
   module AutoInstrument
@@ -31,160 +35,10 @@ module ScoutApm
       end
 
       def self.rewrite(path, code = nil)
-        code ||= File.read(path)
-
-        ast = ::Parser::CurrentRuby.parse(code)
-
-        # pp ast
-
-        buffer = ::Parser::Source::Buffer.new(path)
-        buffer.source = code
-
-        rewriter = Rewriter.new
-
-        # Rewrite the AST, returns a String with the new form.
-        rewriter.rewrite(buffer, ast)
-      end
-
-      class Rewriter < ::Parser::TreeRewriter
-        def initialize
-          super
-
-          # Keeps track of the parent - child relationship between nodes:
-          @nesting = []
-
-          # The stack of method nodes (type :def):
-          @method = []
-
-          # The stack of class nodes:
-          @scope = []
-
-          @cache = Cache.new
-        end
-
-        def instrument(source, file_name, line)
-          # Don't log huge chunks of code... just the first line:
-          if lines = source.lines and lines.count > 1
-            source = lines.first.chomp + "..."
-          end
-
-          method_name = @method.last.children[0]
-          bt = ["#{file_name}:#{line}:in `#{method_name}'"]
-
-          return [
-            "::ScoutApm::AutoInstrument("+ source.dump + ",#{bt}){",
-            "}"
-          ]
-        end
-
-        # Look up 1 or more nodes to check if the parent exists and matches the given type.
-        # @param type [Symbol] the symbol type to match.
-        # @param up [Integer] how far up to look.
-        def parent_type?(type, up = 1)
-          parent = @nesting[@nesting.size - up - 1] and parent.type == type
-        end
-
-        def on_block(node)
-          # If we are not in a method, don't do any instrumentation:
-          return if @method.empty?
-
-          line = node.location.line || 'line?'
-          column = node.location.column || 'column?' # not used
-          method_name = node.children[0].children[1] || '*unknown*' # not used
-          file_name = @source_rewriter.source_buffer.name
-
-          wrap(node.location.expression, *instrument(node.location.expression.source, file_name, line))
-        end
-
-        def on_mlhs(node)
-          # Ignore / don't instrument multiple assignment (LHS).
-          return
-        end
-
-        def on_op_asgn(node)
-          process(node.children[2])
-        end
-
-        def on_or_asgn(node)
-          process(node.children[1])
-        end
-
-        def on_and_asgn(node)
-          process(node.children[1])
-        end
-
-        # Handle the method call AST node. If this method doesn't call `super`, no futher rewriting is applied to children.
-        def on_send(node)
-          # We aren't interested in top level function calls:
-          return if @method.empty?
-
-          if @cache.local_assignments?(node)
-            return super
-          end
-
-          # This ignores both initial block method invocation `*x*{}`, and subsequent nested invocations `x{*y*}`:
-          return if parent_type?(:block)
-
-          # Extract useful metadata for instrumentation:
-          line = node.location.line || 'line?'
-          column = node.location.column || 'column?' # not used
-          method_name = node.children[1] || '*unknown*' # not used
-          file_name = @source_rewriter.source_buffer.name
-
-          # Wrap the expression with instrumentation:
-          wrap(node.location.expression, *instrument(node.location.expression.source, file_name, line))
-        end
-
-        def on_hash(node)
-          node.children.each do |pair|
-            # Skip `pair` if we're sure it's not using the hash shorthand syntax
-            next if pair.type != :pair
-            key_node, value_node = pair.children
-            next unless key_node.type == :sym && value_node.type == :send
-            key = key_node.children[0]
-            next unless value_node.children.size == 2 && value_node.children[0].nil? && key == value_node.children[1]
-
-            # Extract useful metadata for instrumentation:
-            line = pair.location.line || 'line?'
-            # column = pair.location.column || 'column?' # not used
-            # method_name = key || '*unknown*' # not used
-            file_name = @source_rewriter.source_buffer.name
-
-            instrument_before, instrument_after = instrument(pair.location.expression.source, file_name, line)
-            replace(pair.loc.expression, "#{key}: #{instrument_before}#{key}#{instrument_after}")
-          end
-          super
-        end
-
-        # def on_class(node)
-        #   class_name = node.children[1]
-        #
-        #   Kernel.const_get(class_name).ancestors.include? ActionController::Controller
-        #
-        #   if class_name =~ /.../
-        #     super # continue processing
-        #   end
-        # end
-
-        # Invoked for every AST node as it is processed top to bottom.
-        def process(node)
-          # We are nesting inside this node:
-          @nesting.push(node)
-
-          if node and node.type == :def
-            # If the node is a method, push it on the method stack as well:
-            @method.push(node)
-            super
-            @method.pop
-          elsif node and node.type == :class
-            @scope.push(node.children[0])
-            super
-            @scope.pop
-          else
-            super
-          end
-
-          @nesting.pop
+        if defined?(Prism)
+          PrismImplementation.rewrite(path, code)
+        else
+          ParserImplementation.rewrite(path, code)
         end
       end
     end

data/lib/scout_apm/auto_instrument/requirements.rb

@@ -0,0 +1,11 @@
+begin
+  require 'prism'
+rescue LoadError
+end
+
+unless defined?(Prism)
+  begin
+    require 'parser'
+  rescue LoadError
+  end
+end

data/lib/scout_apm/background_job_integrations/delayed_job.rb

@@ -4,6 +4,7 @@ module ScoutApm
       ACTIVE_JOB_KLASS = 'ActiveJob::QueueAdapters::DelayedJobAdapter::JobWrapper'.freeze
       DJ_PERFORMABLE_METHOD = 'Delayed::PerformableMethod'.freeze
 
+
       attr_reader :logger
 
       def name
@@ -69,8 +70,31 @@ module ScoutApm
 
                 # Call the job itself.
                 block.call(job, *args)
-              rescue
+              rescue Exception => exception
+                # Capture the error for further processing and shipping
                 req.error!
+                # Abusing this key to pass job info
+                params_key = 'action_dispatch.request.parameters'
+                env = {}
+
+                # Get job data safely - check for job_data first (ActiveJob), then fall back to args (PerformableMethod)
+                env[params_key] = if job.payload_object.respond_to?(:job_data)
+                                    job.payload_object.job_data
+                                  elsif job.payload_object.respond_to?(:args)
+                                    # For PerformableMethod, create a hash with relevant info
+                                    {
+                                      'args' => job.payload_object.args,
+                                      'method_name' => job.payload_object.method_name,
+                                      'object' => job.payload_object.object.class.to_s
+                                    }
+                                  else
+                                    {}
+                                  end
+
+                env[:custom_controller] = name
+                env[:custom_action] = queue
+                context = ScoutApm::Agent.instance.context
+                context.error_buffer.capture(exception, env)
                 raise
               ensure
                 req.stop_layer if started_job

data/lib/scout_apm/background_job_integrations/faktory.rb

@@ -60,8 +60,14 @@ module ScoutApm
           started_job = true
 
           yield
-        rescue
+        rescue Exception => exception
           req.error!
+          env = {
+            :custom_controller => job_class(job),
+            :custom_action => queue
+          }
+          context = ScoutApm::Agent.instance.context
+          context.error_buffer.capture(exception, env)
           raise
         ensure
           req.stop_layer if started_job

data/lib/scout_apm/background_job_integrations/good_job.rb

@@ -34,8 +34,14 @@ module ScoutApm
               started_job = true # Following Convention
 
               block.call
-            rescue
+            rescue Exception => exception
               req.error!
+              env = {
+                :custom_controller => job.class.name,
+                :custom_action => job.queue_name.presence || UNKNOWN_QUEUE_PLACEHOLDER
+              }
+              context = ScoutApm::Agent.instance.context
+              context.error_buffer.capture(exception, env)
               raise
             ensure
               req.stop_layer if started_job

data/lib/scout_apm/background_job_integrations/legacy_sneakers.rb

@@ -42,8 +42,14 @@ module ScoutApm
           else
             super
           end
-        rescue Exception
+        rescue Exception => exception
           req.error!
+          env = {
+            :custom_controller => job_class,
+            :custom_action => queue
+          }
+          context = ScoutApm::Agent.instance.context
+          context.error_buffer.capture(exception, env)
           raise
         ensure
           req.stop_layer if started_job

data/lib/scout_apm/background_job_integrations/que.rb

@@ -115,8 +115,14 @@ module ScoutApm
               started_job = true
 
               _run_without_scout(*args)
-            rescue Exception => e
+            rescue Exception => exception
               req.error!
+              env = {
+                :custom_controller => job_class,
+                :custom_action => queue
+              }
+              context = ScoutApm::Agent.instance.context
+              context.error_buffer.capture(exception, env)
               raise
             ensure
               req.stop_layer if started_job

data/lib/scout_apm/background_job_integrations/shoryuken.rb

@@ -79,8 +79,14 @@ module ScoutApm
           started_job = true
 
           yield
-        rescue Exception => e
+        rescue Exception => exception
           req.error!
+          env = {
+            :custom_controller => job_class,
+            :custom_action => queue
+          }
+          context = ScoutApm::Agent.instance.context
+          context.error_buffer.capture(exception, env)
           raise
         ensure
           req.stop_layer if started_job

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -52,11 +52,14 @@ module ScoutApm
       def call(_worker, msg, queue)
         req = ScoutApm::RequestManager.lookup
         req.annotate_request(:queue_latency => latency(msg))
+        class_name = job_class(msg)
+
+        add_context!(msg, class_name) if capture_job_args?
 
         begin
           req.start_layer(ScoutApm::Layer.new('Queue', queue))
           started_queue = true
-          req.start_layer(ScoutApm::Layer.new('Job', job_class(msg)))
+          req.start_layer(ScoutApm::Layer.new('Job', class_name))
           started_job = true
 
           yield
@@ -129,6 +132,41 @@ module ScoutApm
         UNKNOWN_CLASS_PLACEHOLDER
       end
 
+      def capture_job_args?
+        ScoutApm::Agent.instance.context.config.value("job_params_capture")
+      end
+
+      def add_context!(msg, class_name)
+        return if class_name == UNKNOWN_CLASS_PLACEHOLDER
+        
+        klass = class_name.constantize rescue nil
+        return if klass.nil?
+
+        # Only allow required and optional parameters, as others aren't fully supported by Sidekiq by default.
+        # This also keeps it easy in terms of the canonical signature of parameters.
+        allowed_parameter_types = [:req, :opt]
+
+        known_parameters =
+          klass.instance_method(:perform).parameters.each_with_object([]) do |(type, name), acc|
+            acc << name if allowed_parameter_types.include?(type)
+          end
+
+        return if known_parameters.empty?
+
+        arguments = msg.fetch('args', [])
+        
+        # Don't think this can actually happen. With perform_all_later, 
+        # it appears we go through this middleware individually (even with multiples of the same job type).
+        return if arguments.length > 1
+
+        job_args = arguments.first.fetch('arguments', [])
+
+        # Reduce known parameters to just the ones that are present in the job arguments (excluding non altered optional params)
+        known_parameters = known_parameters[0...job_args.length]
+
+        ScoutApm::Context.add(filter_params(known_parameters.zip(job_args).to_h))
+      end
+
       def latency(msg, time = Time.now.to_f)
         created_at = msg['enqueued_at'] || msg['created_at']
         if created_at
@@ -146,6 +184,56 @@ module ScoutApm
       rescue
         0
       end
+
+      ###################
+      # Filtering Params
+      ###################
+
+      # Replaces parameter values with a string / set in config file
+      def filter_params(params)
+        return params unless filtered_params_config
+
+        params.each do |k, v|
+          if filter_key?(k)
+            params[k] = "[FILTERED]"
+            next
+          end
+
+          if filter_value?(v)
+            params[k] = "[UNSUPPORTED TYPE]"
+          end
+        end
+
+        params
+      end
+
+      def filter_value?(value)
+        !ScoutApm::Context::VALID_TYPES.any? { |klass| value.is_a?(klass) }
+      end
+
+      # Check, if a key should be filtered
+      def filter_key?(key)
+        params_to_filter.any? do |filter|
+          key.to_s == filter.to_s # key.to_s.include?(filter.to_s)
+        end
+      end
+
+      def params_to_filter
+        @params_to_filter ||= filtered_params_config + rails_filtered_params
+      end
+
+      # TODO: Flip this over to use a new class like filtered exceptions? Some shared logic between
+      # this and the error service.
+      def filtered_params_config
+        ScoutApm::Agent.instance.context.config.value("job_filtered_params")
+      end
+
+      def rails_filtered_params
+        return [] unless defined?(Rails)
+        Rails.configuration.filter_parameters
+      rescue 
+        []
+      end
     end
   end
 end

data/lib/scout_apm/background_job_integrations/sneakers.rb

@@ -65,8 +65,14 @@ module ScoutApm
               started_job = true
 
               process_work_without_scout(*args)
-            rescue Exception => e
+            rescue Exception => exception
               req.error!
+              env = {
+                :custom_controller => job_class,
+                :custom_action => queue
+              }
+              context = ScoutApm::Agent.instance.context
+              context.error_buffer.capture(exception, env)
               raise
             ensure
               req.stop_layer if started_job

data/lib/scout_apm/background_job_integrations/solid_queue.rb

@@ -32,8 +32,26 @@ module ScoutApm
               started_job = true # Following Convention
 
               block.call
-            rescue
+            rescue Exception => exception
               req.error!
+              # Extract job parameters like DelayedJob does
+              params_key = 'action_dispatch.request.parameters'
+              job_args = begin
+                {
+                  arguments: job.arguments,
+                  job_id: job.job_id,
+                }
+              rescue => e
+                { error_extracting_params: e.message }
+              end
+              
+              env = {
+                params_key => job_args,
+                :custom_controller => job.class.name,
+                :custom_action => job.queue_name.presence || UNKNOWN_QUEUE_PLACEHOLDER
+              }
+              context = ScoutApm::Agent.instance.context
+              context.error_buffer.capture(exception, env)
               raise
             ensure
               req.stop_layer if started_job

data/lib/scout_apm/config.rb

@@ -29,6 +29,8 @@ require 'scout_apm/environment'
 # report_format    - 'json' or 'marshal'. Marshal is legacy and will be removed.
 # scm_subdirectory - if the app root lives in source management in a subdirectory. E.g. #{SCM_ROOT}/src
 # uri_reporting    - 'path' or 'full_path' default is 'full_path', which reports URL params as well as the path.
+# job_params_capture - true/false to enable capturing of job args in the context.
+# job_filtered_params - An array of job argument names to filter/redact out of job reports.
 # record_queue_time - true/false to enable recording of queuetime.
 # remote_agent_host - Internal: What host to bind to, and also send messages to for remote. Default: 127.0.0.1.
 # remote_agent_port - What port to bind the remote webserver to
@@ -44,11 +46,11 @@ require 'scout_apm/environment'
 #                            instruments listed in this array. Default: []
 # ignore_endpoints         - An array of endpoints to ignore. These are matched as regular expressions. (supercedes 'ignore')
 # ignore_jobs              - An array of job names to ignore.
-# sample_rate              - Rate to sample entire application. An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent.
-# sample_endpoints         - An array of endpoints to sample. These are matched as regular expressions with individual sample rate of 0 to 100.
-# sample_jobs              - An array of job names with individual sample rate of 0 to 100.
-# endpoint_sample_rate     - Rate to sample all endpoints. An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent. (supercedes 'sample_rate')
-# job_sample_rate          - Rate to sample all jobs. An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent. (supercedes 'sample_rate')
+# sample_rate              - Rate to sample entire application. A float between 0 and 1. 0 means no requests are tracked, 1 means all are, .05 means 5% are.
+# sample_endpoints         - An array of endpoints to sample. These are matched as regular expressions with individual sample rate of 0 to 1.
+# sample_jobs              - An array of job names with individual sample rate of 0 to 1.
+# endpoint_sample_rate     - Rate to sample all endpoints. A float between 0 and 1. 0 means no requests are tracked, 1 means all. (supercedes 'sample_rate')
+# job_sample_rate          - Rate to sample all jobs. A float between 0 and 1. 0 means no requests are tracked, 1 means all. (supercedes 'sample_rate')
 #
 #
 # Errors Service Configuration
@@ -96,6 +98,8 @@ module ScoutApm
         'profile',
         'proxy',
         'record_queue_time',
+        'job_params_capture',
+        'job_filtered_params',
         'remote_agent_host',
         'remote_agent_port',
         'report_format',
@@ -207,6 +211,23 @@ module ScoutApm
       end
     end
 
+    class SampleRateCoercion
+      def coerce(val)
+        v = val.to_f
+        # Anything above 1 is assumed a percentage for backwards compat, so convert to a decimal
+        if v > 1
+          v = v / 100
+        end
+        if v < 0 || v > 1
+          v = v.clamp(0, 1)
+        end
+        v
+      end
+    end
+
+    # Map of config keys to coercions.  Any key not listed here will be passed
+    # through without modification.
+
 
     SETTING_COERCIONS = {
       'async_recording' => BooleanCoercion.new,
@@ -226,7 +247,9 @@ module ScoutApm
       'external_service_metric_report_limit' => IntegerCoercion.new,
       'instrument_http_url_length' => IntegerCoercion.new,
       'record_queue_time' => BooleanCoercion.new,
-      'sample_rate' => IntegerCoercion.new,
+      'job_params_capture' => BooleanCoercion.new,
+      'job_filtered_params' => JsonCoercion.new,
+      'sample_rate' => SampleRateCoercion.new,
       'sample_endpoints' => JsonCoercion.new,
       'sample_jobs' => JsonCoercion.new,
       'endpoint_sample_rate' => NullableIntegerCoercion.new,
@@ -357,13 +380,15 @@ module ScoutApm
         'external_service_metric_limit'        => 5000, # The hard limit on external service metrics
         'external_service_metric_report_limit' => 1000,
         'instrument_http_url_length'           => 300,
-        'sample_rate'                          => 100,
+        'sample_rate'                          => 1,
         'sample_endpoints'                     => [],
         'sample_jobs'                          => [],
         'endpoint_sample_rate'                 => nil,
         'job_sample_rate'                      => nil,
         'start_resque_server_instrument'       => true, # still only starts if Resque is detected
         'collect_remote_ip'                    => true,
+        'job_params_capture'                   => false,
+        'job_filtered_params'                  => [],
         'record_queue_time'                    => true,
         'timeline_traces'                      => true,
         'auto_instruments'                     => false,

data/lib/scout_apm/context.rb

@@ -7,6 +7,8 @@ module ScoutApm
   class Context
     attr_reader :context
 
+    VALID_TYPES = [String, Symbol, Numeric, Time, Date, TrueClass, FalseClass]
+
     def initialize(context)
       @context = context
       @extra = {}
@@ -93,7 +95,7 @@ module ScoutApm
       value = key_value.values.last
       if value.nil?
         false # don't log this ... easy to happen
-      elsif !valid_type?([String, Symbol, Numeric, Time, Date, TrueClass, FalseClass],value)
+      elsif !valid_type?(VALID_TYPES, value)
         logger.info "The value for [#{key_value.keys.first}] is not a valid type [#{value.class}]."
         false
       else

data/lib/scout_apm/error_service/error_record.rb

@@ -18,10 +18,12 @@ module ScoutApm
         @agent_context = agent_context
 
         @context = if context
-          context.to_hash
+          context.to_flat_hash
         else
           {}
         end
+        # Add the transaction_id, as it won't be added to the context normally until the request has been recorded.
+        @context[:transaction_id] ||= RequestManager.lookup.transaction_id
 
         @exception_class = LengthLimit.new(exception.class.name).to_s
         @message = LengthLimit.new(exception.message, 100).to_s
@@ -46,6 +48,7 @@ module ScoutApm
         # For background workers like sidekiq
         # TODO: extract data creation for background jobs
         components[:controller] ||= env[:custom_controller]
+        components[:action] ||= env[:custom_action]
 
         components
       end
@@ -92,6 +95,7 @@ module ScoutApm
 
       # Capture params from env
       KEYS_TO_KEEP = [
+        "REQUEST_METHOD",
         "HTTP_USER_AGENT",
         "HTTP_REFERER",
         "HTTP_ACCEPT_ENCODING",

data/lib/scout_apm/instrument_manager.rb

@@ -33,11 +33,13 @@ module ScoutApm
       install_instrument(ScoutApm::Instruments::Typhoeus)
       install_instrument(ScoutApm::Instruments::HttpClient)
       install_instrument(ScoutApm::Instruments::HTTP)
+      install_instrument(ScoutApm::Instruments::HTTPX)
       install_instrument(ScoutApm::Instruments::Memcached)
       install_instrument(ScoutApm::Instruments::Redis)
       install_instrument(ScoutApm::Instruments::Redis5)
       install_instrument(ScoutApm::Instruments::InfluxDB)
       install_instrument(ScoutApm::Instruments::Elasticsearch)
+      install_instrument(ScoutApm::Instruments::OpenSearch)
       install_instrument(ScoutApm::Instruments::Grape)
     rescue
       logger.warn "Exception loading instruments:"

data/lib/scout_apm/instruments/http_client.rb

@@ -15,8 +15,18 @@ module ScoutApm
       def installed?
         @installed
       end
+      
+      def require_library
+        unless defined?(::HTTPClient)
+          begin
+            require 'httpclient'
+          rescue LoadError
+          end
+        end
+      end
 
       def install(prepend:)
+        require_library
         if defined?(::HTTPClient)
           @installed = true