scout_apm 5.8.0 → 6.0.0

data/lib/scout_apm/auto_instrument/rails.rb

@@ -1,6 +1,10 @@
 
 require 'scout_apm/auto_instrument/layer'
-require 'scout_apm/auto_instrument/parser'
+if defined?(Prism)
+  require 'scout_apm/auto_instrument/prism'
+else
+  require 'scout_apm/auto_instrument/parser'
+end
 
 module ScoutApm
   module AutoInstrument
@@ -31,160 +35,10 @@ module ScoutApm
       end
 
       def self.rewrite(path, code = nil)
-        code ||= File.read(path)
-
-        ast = ::Parser::CurrentRuby.parse(code)
-
-        # pp ast
-
-        buffer = ::Parser::Source::Buffer.new(path)
-        buffer.source = code
-
-        rewriter = Rewriter.new
-
-        # Rewrite the AST, returns a String with the new form.
-        rewriter.rewrite(buffer, ast)
-      end
-
-      class Rewriter < ::Parser::TreeRewriter
-        def initialize
-          super
-
-          # Keeps track of the parent - child relationship between nodes:
-          @nesting = []
-
-          # The stack of method nodes (type :def):
-          @method = []
-
-          # The stack of class nodes:
-          @scope = []
-
-          @cache = Cache.new
-        end
-
-        def instrument(source, file_name, line)
-          # Don't log huge chunks of code... just the first line:
-          if lines = source.lines and lines.count > 1
-            source = lines.first.chomp + "..."
-          end
-
-          method_name = @method.last.children[0]
-          bt = ["#{file_name}:#{line}:in `#{method_name}'"]
-
-          return [
-            "::ScoutApm::AutoInstrument("+ source.dump + ",#{bt}){",
-            "}"
-          ]
-        end
-
-        # Look up 1 or more nodes to check if the parent exists and matches the given type.
-        # @param type [Symbol] the symbol type to match.
-        # @param up [Integer] how far up to look.
-        def parent_type?(type, up = 1)
-          parent = @nesting[@nesting.size - up - 1] and parent.type == type
-        end
-
-        def on_block(node)
-          # If we are not in a method, don't do any instrumentation:
-          return if @method.empty?
-
-          line = node.location.line || 'line?'
-          column = node.location.column || 'column?' # not used
-          method_name = node.children[0].children[1] || '*unknown*' # not used
-          file_name = @source_rewriter.source_buffer.name
-
-          wrap(node.location.expression, *instrument(node.location.expression.source, file_name, line))
-        end
-
-        def on_mlhs(node)
-          # Ignore / don't instrument multiple assignment (LHS).
-          return
-        end
-
-        def on_op_asgn(node)
-          process(node.children[2])
-        end
-
-        def on_or_asgn(node)
-          process(node.children[1])
-        end
-
-        def on_and_asgn(node)
-          process(node.children[1])
-        end
-
-        # Handle the method call AST node. If this method doesn't call `super`, no futher rewriting is applied to children.
-        def on_send(node)
-          # We aren't interested in top level function calls:
-          return if @method.empty?
-
-          if @cache.local_assignments?(node)
-            return super
-          end
-
-          # This ignores both initial block method invocation `*x*{}`, and subsequent nested invocations `x{*y*}`:
-          return if parent_type?(:block)
-
-          # Extract useful metadata for instrumentation:
-          line = node.location.line || 'line?'
-          column = node.location.column || 'column?' # not used
-          method_name = node.children[1] || '*unknown*' # not used
-          file_name = @source_rewriter.source_buffer.name
-
-          # Wrap the expression with instrumentation:
-          wrap(node.location.expression, *instrument(node.location.expression.source, file_name, line))
-        end
-
-        def on_hash(node)
-          node.children.each do |pair|
-            # Skip `pair` if we're sure it's not using the hash shorthand syntax
-            next if pair.type != :pair
-            key_node, value_node = pair.children
-            next unless key_node.type == :sym && value_node.type == :send
-            key = key_node.children[0]
-            next unless value_node.children.size == 2 && value_node.children[0].nil? && key == value_node.children[1]
-
-            # Extract useful metadata for instrumentation:
-            line = pair.location.line || 'line?'
-            # column = pair.location.column || 'column?' # not used
-            # method_name = key || '*unknown*' # not used
-            file_name = @source_rewriter.source_buffer.name
-
-            instrument_before, instrument_after = instrument(pair.location.expression.source, file_name, line)
-            replace(pair.loc.expression, "#{key}: #{instrument_before}#{key}#{instrument_after}")
-          end
-          super
-        end
-
-        # def on_class(node)
-        #   class_name = node.children[1]
-        #
-        #   Kernel.const_get(class_name).ancestors.include? ActionController::Controller
-        #
-        #   if class_name =~ /.../
-        #     super # continue processing
-        #   end
-        # end
-
-        # Invoked for every AST node as it is processed top to bottom.
-        def process(node)
-          # We are nesting inside this node:
-          @nesting.push(node)
-
-          if node and node.type == :def
-            # If the node is a method, push it on the method stack as well:
-            @method.push(node)
-            super
-            @method.pop
-          elsif node and node.type == :class
-            @scope.push(node.children[0])
-            super
-            @scope.pop
-          else
-            super
-          end
-
-          @nesting.pop
+        if defined?(Prism)
+          PrismImplementation.rewrite(path, code)
+        else
+          ParserImplementation.rewrite(path, code)
         end
       end
     end

data/lib/scout_apm/auto_instrument/requirements.rb

@@ -0,0 +1,11 @@
+begin
+  require 'prism'
+rescue LoadError
+end
+
+unless defined?(Prism)
+  begin
+    require 'parser'
+  rescue LoadError
+  end
+end

data/lib/scout_apm/background_job_integrations/delayed_job.rb

@@ -76,7 +76,21 @@ module ScoutApm
                 # Abusing this key to pass job info
                 params_key = 'action_dispatch.request.parameters'
                 env = {}
-                env[params_key] = job.payload_object.job_data
+
+                # Get job data safely - check for job_data first (ActiveJob), then fall back to args (PerformableMethod)
+                env[params_key] = if job.payload_object.respond_to?(:job_data)
+                                    job.payload_object.job_data
+                                  elsif job.payload_object.respond_to?(:args)
+                                    # For PerformableMethod, create a hash with relevant info
+                                    {
+                                      'args' => job.payload_object.args,
+                                      'method_name' => job.payload_object.method_name,
+                                      'object' => job.payload_object.object.class.to_s
+                                    }
+                                  else
+                                    {}
+                                  end
+
                 env[:custom_controller] = name
                 env[:custom_action] = queue
                 context = ScoutApm::Agent.instance.context

data/lib/scout_apm/background_job_integrations/sidekiq.rb

@@ -52,11 +52,14 @@ module ScoutApm
       def call(_worker, msg, queue)
         req = ScoutApm::RequestManager.lookup
         req.annotate_request(:queue_latency => latency(msg))
+        class_name = job_class(msg)
+
+        add_context!(msg, class_name) if capture_job_args?
 
         begin
           req.start_layer(ScoutApm::Layer.new('Queue', queue))
           started_queue = true
-          req.start_layer(ScoutApm::Layer.new('Job', job_class(msg)))
+          req.start_layer(ScoutApm::Layer.new('Job', class_name))
           started_job = true
 
           yield
@@ -129,6 +132,41 @@ module ScoutApm
         UNKNOWN_CLASS_PLACEHOLDER
       end
 
+      def capture_job_args?
+        ScoutApm::Agent.instance.context.config.value("job_params_capture")
+      end
+
+      def add_context!(msg, class_name)
+        return if class_name == UNKNOWN_CLASS_PLACEHOLDER
+        
+        klass = class_name.constantize rescue nil
+        return if klass.nil?
+
+        # Only allow required and optional parameters, as others aren't fully supported by Sidekiq by default.
+        # This also keeps it easy in terms of the canonical signature of parameters.
+        allowed_parameter_types = [:req, :opt]
+
+        known_parameters =
+          klass.instance_method(:perform).parameters.each_with_object([]) do |(type, name), acc|
+            acc << name if allowed_parameter_types.include?(type)
+          end
+
+        return if known_parameters.empty?
+
+        arguments = msg.fetch('args', [])
+        
+        # Don't think this can actually happen. With perform_all_later, 
+        # it appears we go through this middleware individually (even with multiples of the same job type).
+        return if arguments.length > 1
+
+        job_args = arguments.first.fetch('arguments', [])
+
+        # Reduce known parameters to just the ones that are present in the job arguments (excluding non altered optional params)
+        known_parameters = known_parameters[0...job_args.length]
+
+        ScoutApm::Context.add(filter_params(known_parameters.zip(job_args).to_h))
+      end
+
       def latency(msg, time = Time.now.to_f)
         created_at = msg['enqueued_at'] || msg['created_at']
         if created_at
@@ -146,6 +184,56 @@ module ScoutApm
       rescue
         0
       end
+
+      ###################
+      # Filtering Params
+      ###################
+
+      # Replaces parameter values with a string / set in config file
+      def filter_params(params)
+        return params unless filtered_params_config
+
+        params.each do |k, v|
+          if filter_key?(k)
+            params[k] = "[FILTERED]"
+            next
+          end
+
+          if filter_value?(v)
+            params[k] = "[UNSUPPORTED TYPE]"
+          end
+        end
+
+        params
+      end
+
+      def filter_value?(value)
+        !ScoutApm::Context::VALID_TYPES.any? { |klass| value.is_a?(klass) }
+      end
+
+      # Check, if a key should be filtered
+      def filter_key?(key)
+        params_to_filter.any? do |filter|
+          key.to_s == filter.to_s # key.to_s.include?(filter.to_s)
+        end
+      end
+
+      def params_to_filter
+        @params_to_filter ||= filtered_params_config + rails_filtered_params
+      end
+
+      # TODO: Flip this over to use a new class like filtered exceptions? Some shared logic between
+      # this and the error service.
+      def filtered_params_config
+        ScoutApm::Agent.instance.context.config.value("job_filtered_params")
+      end
+
+      def rails_filtered_params
+        return [] unless defined?(Rails)
+        Rails.configuration.filter_parameters
+      rescue 
+        []
+      end
     end
   end
 end

data/lib/scout_apm/config.rb

@@ -29,6 +29,8 @@ require 'scout_apm/environment'
 # report_format    - 'json' or 'marshal'. Marshal is legacy and will be removed.
 # scm_subdirectory - if the app root lives in source management in a subdirectory. E.g. #{SCM_ROOT}/src
 # uri_reporting    - 'path' or 'full_path' default is 'full_path', which reports URL params as well as the path.
+# job_params_capture - true/false to enable capturing of job args in the context.
+# job_filtered_params - An array of job argument names to filter/redact out of job reports.
 # record_queue_time - true/false to enable recording of queuetime.
 # remote_agent_host - Internal: What host to bind to, and also send messages to for remote. Default: 127.0.0.1.
 # remote_agent_port - What port to bind the remote webserver to
@@ -44,11 +46,11 @@ require 'scout_apm/environment'
 #                            instruments listed in this array. Default: []
 # ignore_endpoints         - An array of endpoints to ignore. These are matched as regular expressions. (supercedes 'ignore')
 # ignore_jobs              - An array of job names to ignore.
-# sample_rate              - Rate to sample entire application. An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent.
-# sample_endpoints         - An array of endpoints to sample. These are matched as regular expressions with individual sample rate of 0 to 100.
-# sample_jobs              - An array of job names with individual sample rate of 0 to 100.
-# endpoint_sample_rate     - Rate to sample all endpoints. An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent. (supercedes 'sample_rate')
-# job_sample_rate          - Rate to sample all jobs. An integer between 0 and 100. 0 means no traces are sent, 100 means all traces are sent. (supercedes 'sample_rate')
+# sample_rate              - Rate to sample entire application. A float between 0 and 1. 0 means no requests are tracked, 1 means all are, .05 means 5% are.
+# sample_endpoints         - An array of endpoints to sample. These are matched as regular expressions with individual sample rate of 0 to 1.
+# sample_jobs              - An array of job names with individual sample rate of 0 to 1.
+# endpoint_sample_rate     - Rate to sample all endpoints. A float between 0 and 1. 0 means no requests are tracked, 1 means all. (supercedes 'sample_rate')
+# job_sample_rate          - Rate to sample all jobs. A float between 0 and 1. 0 means no requests are tracked, 1 means all. (supercedes 'sample_rate')
 #
 #
 # Errors Service Configuration
@@ -96,6 +98,8 @@ module ScoutApm
         'profile',
         'proxy',
         'record_queue_time',
+        'job_params_capture',
+        'job_filtered_params',
         'remote_agent_host',
         'remote_agent_port',
         'report_format',
@@ -207,6 +211,23 @@ module ScoutApm
       end
     end
 
+    class SampleRateCoercion
+      def coerce(val)
+        v = val.to_f
+        # Anything above 1 is assumed a percentage for backwards compat, so convert to a decimal
+        if v > 1
+          v = v / 100
+        end
+        if v < 0 || v > 1
+          v = v.clamp(0, 1)
+        end
+        v
+      end
+    end
+
+    # Map of config keys to coercions.  Any key not listed here will be passed
+    # through without modification.
+
 
     SETTING_COERCIONS = {
       'async_recording' => BooleanCoercion.new,
@@ -226,7 +247,9 @@ module ScoutApm
       'external_service_metric_report_limit' => IntegerCoercion.new,
       'instrument_http_url_length' => IntegerCoercion.new,
       'record_queue_time' => BooleanCoercion.new,
-      'sample_rate' => IntegerCoercion.new,
+      'job_params_capture' => BooleanCoercion.new,
+      'job_filtered_params' => JsonCoercion.new,
+      'sample_rate' => SampleRateCoercion.new,
       'sample_endpoints' => JsonCoercion.new,
       'sample_jobs' => JsonCoercion.new,
       'endpoint_sample_rate' => NullableIntegerCoercion.new,
@@ -357,13 +380,15 @@ module ScoutApm
         'external_service_metric_limit'        => 5000, # The hard limit on external service metrics
         'external_service_metric_report_limit' => 1000,
         'instrument_http_url_length'           => 300,
-        'sample_rate'                          => 100,
+        'sample_rate'                          => 1,
         'sample_endpoints'                     => [],
         'sample_jobs'                          => [],
         'endpoint_sample_rate'                 => nil,
         'job_sample_rate'                      => nil,
         'start_resque_server_instrument'       => true, # still only starts if Resque is detected
         'collect_remote_ip'                    => true,
+        'job_params_capture'                   => false,
+        'job_filtered_params'                  => [],
         'record_queue_time'                    => true,
         'timeline_traces'                      => true,
         'auto_instruments'                     => false,

data/lib/scout_apm/context.rb

@@ -7,6 +7,8 @@ module ScoutApm
   class Context
     attr_reader :context
 
+    VALID_TYPES = [String, Symbol, Numeric, Time, Date, TrueClass, FalseClass]
+
     def initialize(context)
       @context = context
       @extra = {}
@@ -93,7 +95,7 @@ module ScoutApm
       value = key_value.values.last
       if value.nil?
         false # don't log this ... easy to happen
-      elsif !valid_type?([String, Symbol, Numeric, Time, Date, TrueClass, FalseClass],value)
+      elsif !valid_type?(VALID_TYPES, value)
         logger.info "The value for [#{key_value.keys.first}] is not a valid type [#{value.class}]."
         false
       else

data/lib/scout_apm/error_service/error_record.rb

@@ -18,7 +18,7 @@ module ScoutApm
         @agent_context = agent_context
 
         @context = if context
-          context.to_hash
+          context.to_flat_hash
         else
           {}
         end

data/lib/scout_apm/instrument_manager.rb

@@ -33,11 +33,13 @@ module ScoutApm
       install_instrument(ScoutApm::Instruments::Typhoeus)
       install_instrument(ScoutApm::Instruments::HttpClient)
       install_instrument(ScoutApm::Instruments::HTTP)
+      install_instrument(ScoutApm::Instruments::HTTPX)
       install_instrument(ScoutApm::Instruments::Memcached)
       install_instrument(ScoutApm::Instruments::Redis)
       install_instrument(ScoutApm::Instruments::Redis5)
       install_instrument(ScoutApm::Instruments::InfluxDB)
       install_instrument(ScoutApm::Instruments::Elasticsearch)
+      install_instrument(ScoutApm::Instruments::OpenSearch)
       install_instrument(ScoutApm::Instruments::Grape)
     rescue
       logger.warn "Exception loading instruments:"

data/lib/scout_apm/instruments/http_client.rb

@@ -15,8 +15,18 @@ module ScoutApm
       def installed?
         @installed
       end
+      
+      def require_library
+        unless defined?(::HTTPClient)
+          begin
+            require 'httpclient'
+          rescue LoadError
+          end
+        end
+      end
 
       def install(prepend:)
+        require_library
         if defined?(::HTTPClient)
           @installed = true
 

data/lib/scout_apm/instruments/httpx.rb

@@ -0,0 +1,119 @@
+module ScoutApm
+  module Instruments
+    class HTTPX
+      attr_reader :context
+
+      def initialize(context)
+        @context = context
+        @installed = false
+      end
+
+      def logger
+        context.logger
+      end
+
+      def installed?
+        @installed
+      end
+
+      def install(prepend:)
+        if defined?(::HTTPX) && defined?(::HTTPX::Session)
+          @installed = true
+
+          logger.info "Instrumenting HTTPX"
+
+          ::HTTPX::Session.send(:prepend, HTTPXInstrumentationPrepend)
+        end
+      end
+
+      module HTTPXInstrumentationPrepend
+        def request(*args, **params)
+          verb, desc = determine_verb_and_desc(*args, **params)
+
+          layer = ScoutApm::Layer.new("HTTP", verb)
+          layer.desc = desc
+
+          req = ScoutApm::RequestManager.lookup
+          req.start_layer(layer)
+
+          begin
+            super(*args, **params)
+          ensure
+            req.stop_layer
+          end
+        end
+
+        private
+
+        # See the following for various argument patterns:
+        # https://gitlab.com/os85/httpx/-/blob/v1.6.3/lib/httpx/session.rb?ref_type=tags#L87
+        def determine_verb_and_desc(*args, **params)
+          # Pattern 1: session.request(req1) or session.request(req1, req2, ...)
+          if args.first.is_a?(::HTTPX::Request)
+            if args.length > 1
+              return args.first.verb.to_s.upcase, "#{args.length} requests"
+            else
+              return args.first.verb.to_s.upcase, scout_url_desc(args.first.uri)
+            end
+          end
+
+          # Pattern 2: session.request("GET", "https://server.org/a")
+          # Pattern 3: session.request("GET", ["https://server.org/a", "https://server.org/b"])
+          # Pattern 4: session.request("POST", ["https://server.org/a"], form: { ... })
+          # Pattern 5: session.request("GET", ["https://..."], headers: { ... })
+          if args.first.is_a?(String) || args.first.is_a?(Symbol)
+            verb = args.first.to_s.upcase
+
+            if args[1].is_a?(String)
+              return verb, scout_url_desc(args[1])
+            elsif args[1].is_a?(Array)
+              return verb, scout_url_desc(args[1][0]) unless args[1].length > 1
+              return verb, "#{args[1].length} requests"
+            else
+              return verb, ""
+            end
+          end
+
+          # Pattern 6: session.request(["GET", "https://..."], ["GET", "https://..."])
+          # Pattern 7: session.request(["POST", "https://...", form: {...}], ["GET", "https://..."])
+          if args.first.is_a?(Array)
+            if args.length > 1
+              verb = args.first[0].to_s.upcase rescue "REQUEST"
+              return verb, "#{args.length} requests"
+            elsif args.first.length >= 2
+              verb = args.first[0].to_s.upcase rescue "REQUEST"
+              url = args.first[1]
+              return verb, scout_url_desc(url)
+            end
+          end
+
+          return "REQUEST", ""
+        end
+
+        def scout_url_desc(uri)
+          max_length = ScoutApm::Agent.instance.context.config.value('instrument_http_url_length')
+          uri_str = uri.to_s
+
+          # URI object
+          if uri.respond_to?(:host) && uri.respond_to?(:path)
+            path = uri.path.to_s
+            path = "/" if path.empty?
+            result = "#{uri.host}#{path.split('?').first}"
+          # String URL
+          elsif uri_str =~ %r{^https?://([^/]+)(/[^?]*)?}
+            host = $1
+            path = $2 || "/"
+            result = "#{host}#{path}"
+          else
+            # Fallback
+            result = uri_str.split('?').first
+          end
+
+          result[0..(max_length - 1)]
+        rescue => e
+          ""
+        end
+      end
+    end
+  end
+end