scout_apm 5.8.0 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80d2cac0818a8487fe1e9ca2395faccbbbadf2be368151e636e39dd7188b34ea
-  data.tar.gz: 1c46ee7b70ac1bd2e9ff4ed4232c7304edc02088563df840619aedcb5e43384d
+  metadata.gz: 4c8fea1cfd66466534fa0ca5e7aaac6cdb332156e02a4e933d6fb883eb9fd68e
+  data.tar.gz: d6f8ab26c4a7d77cde8f0f4fce2b1ef4f849635e839169fe9163e9b64b796294
 SHA512:
-  metadata.gz: da2b3e51bea0eb7198d76edd997ef62da6719b3e0df7bd7093f56cbd51d2f0426f6798ef57d641e786a3a16732ad6e9ad141ce977fce0d9888541ff72bfbb11d
-  data.tar.gz: 42e1bb5874b213da9c8be9b4024263737bf8c34ae7c968b2259939f87bb3f983a105036d6cd3222de02964b8f40aedab8f5e47f10248ed5053c517acddf67833
+  metadata.gz: 1d38a64cd133729432a38611167a4551e5ebb208eeddd1d346a5622d0484ba82a4fedb1ac14e385cc48671cd1902726350662202db35a88a28cbde9fb7613702
+  data.tar.gz: 7755eb50a09652dba9fa4168893f51f3684b0f14c3fa681c8b2b0af3473eeb9d5b1cec98da46e49612b9f490bc5e9355a187d8658ced8616af0d6d0ad2bdef25

data/.github/workflows/test.yml

@@ -53,6 +53,8 @@ jobs:
             gemfile: gems/sqlite3-v2.gemfile
           - ruby: 3.3
             gemfile: gems/sqlite3-v2.gemfile
+          - ruby: 3.4
+            gemfile: gems/sqlite3-v2.gemfile
     env:
       BUNDLE_GEMFILE: ${{ matrix.gemfile }}
       SCOUT_TEST_FEATURES: ${{ matrix.test_features }}

data/CHANGELOG.markdown

@@ -1,5 +1,16 @@
-# Unreleased
-- Enable Delayed Job error tracking (#565)
+# 6.0.0
+- Add Prism AutoInstruments Support (#582) (#594)
+- Add OpenSearch instrumentation (#563)
+- Add HTTPX instrumentation (#588)
+- Add ability to automatically capture Sidekiq job args as context
+  - `job_params_capture` - Set to true to enable job argument capturing
+  - `job_params_filter`  - A list of arguments to filter (automatically includes Rails filtered_parameters) 
+- Fix user error context being incorrectly flattened (#581)
+- Handle Delayed Job PerformableMethod jobs for error tracking (#584)
+- Require 'httpclient' library on instrumentation install (#586)
+- Add support for Puma 7.0 hook format (#589)
+- Update sample rate to support floats (#564)
+- Reintegrate Regex support for ignored URIs (#574)
 
 # 5.8.0
 - Add error monitoring to SolidQueue, faktory, goodjob, que, shoryuken, sneakers (#571)
@@ -7,6 +18,7 @@
 - Remove unused time util conflict with DeviseTokenAuth (#575)
 - Add transaction ID to error records (#568)-
 - Adds request method to captured error context (#577)
+- Enable Delayed Job error tracking (#565)
 
 # 5.7.1
 - Update error capture API to use context (#560)

data/README.markdown

@@ -2,9 +2,12 @@
 
 [![Build Status](https://github.com/scoutapp/scout_apm_ruby/actions/workflows/test.yml/badge.svg)](https://github.com/scoutapp/scout_apm_ruby/actions)
 
-A Ruby gem for detailed Rails application performance monitoring 📈. Metrics and transaction traces are
-reported to [Scout](https://scoutapp.com), a hosted application monitoring
-service.
+A Ruby gem for detailed Rails application performance monitoring 📈. Metrics, errors and transaction traces are
+reported to [Scout](https://www.scoutapm.com), a hosted application monitoring
+service. We have a free plan for small apps and a 14-day all-access trial to test out all
+the features. If you want to send us Rails logs, add our [other
+gem](https://github.com/scoutapp/scout_apm_ruby_logging) and we will correlate them with
+your performance data!
 
 ## What's the special sauce? 🤔
 
@@ -12,7 +15,6 @@ The Scout agent is engineered to do some wonderful things:
 
 * A unique focus on identifying those hard-to-investigate outliers like memory bloat, N+1s, and user-specific problems. [See an example workflow](http://scoutapp.com/newrelic-alternative).
 * [Low-overhead](http://blog.scoutapp.com/articles/2016/02/07/overhead-benchmarks-new-relic-vs-scout)
-* View your performance metrics during development with [DevTrace](https://docs.scoutapm.com/#devtrace) and in production via [server_timing](https://github.com/scoutapp/ruby_server_timing).
 * Production-Safe profiling of custom code via [ScoutProf](https://docs.scoutapm.com/#scoutprof) (BETA).
 
 ## Getting Started
@@ -29,8 +31,8 @@ Update your Gemfile
 
     bundle install
 
-Signup for a [Scout](https://scoutapm.com) account and put the provided
-config file at `RAILS_ROOT/config/scout_apm.yml`.
+Signup for a [Scout](https://scoutapm.com/users/sign_up?utm_source=github&utm_medium=github&utm_campaign=scout_apm_ruby)
+account and put the provided config file at `RAILS_ROOT/config/scout_apm.yml`.
 
 Your config file should look like:
 
@@ -45,6 +47,16 @@ Your config file should look like:
     production:
       <<: *defaults
 
+## Error Monitoring
+
+All of our accounts include Error Monitoring with 5000 errors/month free. To enable it, add the following to your `scout_apm.yml`:
+
+```yaml
+# Common/dev/production/etc. whereever you would like to start trying it
+# monitor: true should also be required to ensure your App exists in Scout
+errors_enabled: true
+```
+
 ## DevTrace Quickstart
 
 To use DevTrace, our free, no-signup, in-browser development profiler:
@@ -71,8 +83,8 @@ SCOUT_DEV_TRACE=true rails server
 ## Docs
 
 For the complete list of supported frameworks, Rubies, configuration options
-and more, see our [help site](https://docs.scoutapm.com/).
+and more, see our [help site](https://scoutapm.com/docs).
 
 ## Help
 
-Email support@scoutapp.com if you need a hand.
+Email support@scoutapm.com if you need a hand.

data/gems/instruments.gemfile

@@ -6,3 +6,4 @@ gem 'redis'
 gem 'moped'
 gem 'actionpack'
 gem 'actionview'
+gem 'httpx'

data/lib/scout_apm/auto_instrument/instruction_sequence.rb

@@ -7,7 +7,8 @@ module ScoutApm
       def load_iseq(path)
         if Rails.controller_path?(path) & !Rails.ignore?(path)
           begin
-            new_code = Rails.rewrite(path)
+            new_code = Rails.rewrite(path).force_encoding(Encoding.default_external)
+            
             return self.compile(new_code, path, path)
           rescue
             warn "Failed to apply auto-instrumentation to #{path}: #{$!}" if ENV['SCOUT_LOG_LEVEL'].to_s.downcase == "debug"

data/lib/scout_apm/auto_instrument/parser.rb

@@ -4,11 +4,159 @@ raise LoadError, "Parser::TreeRewriter was not defined" unless defined?(Parser::
 
 module ScoutApm
   module AutoInstrument
+    module ParserImplementation
+      def self.rewrite(path, code = nil)
+        code ||= File.read(path)
+
+        ast = ::Parser::CurrentRuby.parse(code)
+
+        buffer = ::Parser::Source::Buffer.new(path)
+        buffer.source = code
+
+        rewriter = Rewriter.new
+
+        # Rewrite the AST, returns a String with the new form.
+        rewriter.rewrite(buffer, ast)
+      end
+
+      class Rewriter < ::Parser::TreeRewriter
+        def initialize
+          super
+
+          # Keeps track of the parent - child relationship between nodes:
+          @nesting = []
+
+          # The stack of method nodes (type :def):
+          @method = []
+
+          # The stack of class nodes:
+          @scope = []
+
+          @cache = Cache.new
+        end
+
+        def instrument(source, file_name, line)
+          # Don't log huge chunks of code... just the first line:
+          if lines = source.lines and lines.count > 1
+            source = lines.first.chomp + "..."
+          end
+
+          method_name = @method.last.children[0]
+          bt = ["#{file_name}:#{line}:in `#{method_name}'"]
+
+          return [
+            "::ScoutApm::AutoInstrument("+ source.dump + ",#{bt}){",
+            "}"
+          ]
+        end
+
+        # Look up 1 or more nodes to check if the parent exists and matches the given type.
+        # @param type [Symbol] the symbol type to match.
+        # @param up [Integer] how far up to look.
+        def parent_type?(type, up = 1)
+          parent = @nesting[@nesting.size - up - 1] and parent.type == type
+        end
+
+        def on_block(node)
+          # If we are not in a method, don't do any instrumentation:
+          return if @method.empty?
+
+          line = node.location.line || 'line?'
+          column = node.location.column || 'column?' # not used
+          method_name = node.children[0].children[1] || '*unknown*' # not used
+          file_name = @source_rewriter.source_buffer.name
+
+          wrap(node.location.expression, *instrument(node.location.expression.source, file_name, line))
+        end
+
+        def on_mlhs(node)
+          # Ignore / don't instrument multiple assignment (LHS).
+          return
+        end
+
+        def on_op_asgn(node)
+          process(node.children[2])
+        end
+
+        def on_or_asgn(node)
+          process(node.children[1])
+        end
+
+        def on_and_asgn(node)
+          process(node.children[1])
+        end
+
+        # Handle the method call AST node. If this method doesn't call `super`, no futher rewriting is applied to children.
+        def on_send(node)
+          # We aren't interested in top level function calls:
+          return if @method.empty?
+
+          if @cache.local_assignments?(node)
+            return super
+          end
+
+          # This ignores both initial block method invocation `*x*{}`, and subsequent nested invocations `x{*y*}`:
+          return if parent_type?(:block)
+
+          # Extract useful metadata for instrumentation:
+          line = node.location.line || 'line?'
+          column = node.location.column || 'column?' # not used
+          method_name = node.children[1] || '*unknown*' # not used
+          file_name = @source_rewriter.source_buffer.name
+
+          # Wrap the expression with instrumentation:
+          wrap(node.location.expression, *instrument(node.location.expression.source, file_name, line))
+        end
+
+        def on_hash(node)
+          node.children.each do |pair|
+            # Skip `pair` if we're sure it's not using the hash shorthand syntax
+            next if pair.type != :pair
+            key_node, value_node = pair.children
+            next unless key_node.type == :sym && value_node.type == :send
+            key = key_node.children[0]
+            next unless value_node.children.size == 2 && value_node.children[0].nil? && key == value_node.children[1]
+
+            # Extract useful metadata for instrumentation:
+            line = pair.location.line || 'line?'
+            # column = pair.location.column || 'column?' # not used
+            # method_name = key || '*unknown*' # not used
+            file_name = @source_rewriter.source_buffer.name
+
+            instrument_before, instrument_after = instrument(pair.location.expression.source, file_name, line)
+            replace(pair.loc.expression, "#{key}: #{instrument_before}#{key}#{instrument_after}")
+          end
+          super
+        end
+
+        # Invoked for every AST node as it is processed top to bottom.
+        def process(node)
+          # We are nesting inside this node:
+          @nesting.push(node)
+
+          if node and node.type == :def
+            # If the node is a method, push it on the method stack as well:
+            @method.push(node)
+            super
+            @method.pop
+          elsif node and node.type == :class
+            @scope.push(node.children[0])
+            super
+            @scope.pop
+          else
+            super
+          end
+
+          @nesting.pop
+        end
+      end
+    end
+
     class Cache
       def initialize
         @local_assignments = {}
       end
-      
+
       def local_assignments?(node)
         unless @local_assignments.key?(node)
           if node.type == :lvasgn
@@ -19,7 +167,7 @@ module ScoutApm
             @local_assignments[node] = false
           end
         end
-        
+
         return @local_assignments[node]
       end
     end

data/lib/scout_apm/auto_instrument/prism.rb

@@ -0,0 +1,357 @@
+
+require 'prism'
+
+module ScoutApm
+  module AutoInstrument
+    module PrismImplementation
+      def self.rewrite(path, code = nil)
+        code ||= File.read(path)
+
+        result = Prism.parse(code)
+        rewriter = Rewriter.new(path, code)
+        rewriter.rewrite(result.value)
+      end
+
+      class Rewriter
+        def initialize(path, code)
+          @path = path
+          # Set encoding to ASCII-8bit as Prism uses byte offsets
+          @code = code.b
+          @replacements = []
+          @instrumented_nodes = Set.new
+
+          # Keeps track of the parent - child relationship between nodes:
+          @nesting = []
+
+          # The stack of method nodes:
+          @method = []
+
+          # The stack of class nodes:
+          @scope = []
+
+          @cache = Cache.new
+        end
+
+        def rewrite(node)
+          process(node)
+          apply_replacements
+        end
+
+        def apply_replacements
+          # Sort replacements by start position in reverse order
+          # This ensures we apply replacements from end to start, avoiding offset issues
+          # when we modify the string
+          sorted_replacements = @replacements.sort_by { |r| -r[:start] }
+
+          result = @code.dup
+          sorted_replacements.each do |replacement|
+            result[replacement[:start]...replacement[:end]] = replacement[:new_text].b
+          end
+          # ::RubyVM::InstructionSequence.compile will infer the encoding when compiling
+          # and will compile with ASCII-8bit correctly.
+          result
+        end
+
+        def add_replacement(start_offset, end_offset, new_text)
+          @replacements << {start: start_offset, end: end_offset, new_text: new_text}
+        end
+
+        def instrument(source, file_name, line)
+          # Don't log huge chunks of code... just the first line:
+          if lines = source.lines and lines.count > 1
+            source = lines.first.chomp + "..."
+          end
+
+          method_name = @method.last.name
+          bt = ["#{file_name}:#{line}:in `#{method_name}'"]
+
+          return [
+            "::ScoutApm::AutoInstrument("+ source.dump + ",#{bt}){",
+            "}"
+          ]
+        end
+
+        # Look up 1 or more nodes to check if the parent exists and matches the given type.
+        # @param type [Class] the node class to match.
+        # @param up [Integer] how far up to look.
+        def parent_type?(type, up = 1)
+          parent = @nesting[@nesting.size - up - 1] and parent.is_a?(type)
+        end
+
+        def wrap_node(node)
+          # Skip if this node or any parent has already been instrumented
+          return if @instrumented_nodes.include?(node)
+
+          # Skip if any ancestor node has been instrumented (to avoid overlapping replacements)
+          @nesting.each do |ancestor|
+            return if @instrumented_nodes.include?(ancestor)
+          end
+
+          # Skip if any descendant node has already been instrumented (to avoid overlapping replacements)
+          # This prevents a parent node from being wrapped when a child node has already been modified
+          return if has_instrumented_descendant?(node)
+
+          start_offset = node.location.start_offset
+          end_offset = node.location.end_offset
+          line = node.location.start_line
+          source = @code[start_offset...end_offset]
+
+          instrument_before, instrument_after = instrument(source, @path, line)
+          new_text = "#{instrument_before}#{source}#{instrument_after}"
+          add_replacement(start_offset, end_offset, new_text)
+          @instrumented_nodes.add(node)
+        end
+
+        def has_instrumented_descendant?(node)
+          node.compact_child_nodes.any? do |child|
+            @instrumented_nodes.include?(child) || has_instrumented_descendant?(child)
+          end
+        end
+
+        def visit_block_node(node)
+          # If we are not in a method, don't do any instrumentation:
+          return process_children(node) if @method.empty?
+
+          # If this block is attached to a CallNode, don't wrap it separately
+          # The CallNode will wrap the entire call including the block
+          return process_children(node) if parent_type?(Prism::CallNode)
+
+          # If this block is attached to a SuperNode or ForwardingSuperNode, don't wrap it separately
+          # The super node will wrap the entire call including the block
+          return process_children(node) if parent_type?(Prism::SuperNode) || parent_type?(Prism::ForwardingSuperNode)
+
+          wrap_node(node)
+        end
+
+        def visit_multi_target_node(node)
+          # Ignore / don't instrument multiple assignment (LHS).
+          return
+        end
+
+        def visit_call_node(node)
+          # We aren't interested in top level function calls:
+          return process_children(node) if @method.empty?
+
+          if @cache.local_assignments?(node)
+            return process_children(node)
+          end
+
+          # This ignores both initial block method invocation and subsequent nested invocations:
+          return process_children(node) if parent_type?(Prism::BlockNode)
+
+          wrap_node(node)
+
+          # Process children to handle nested calls, but blocks attached to this call
+          # won't be wrapped separately (handled by visit_block_node check)
+          process_children(node)
+        end
+
+        def visit_super_node(node)
+          # We aren't interested in top level super calls:
+          return process_children(node) if @method.empty?
+
+          # This ignores super calls inside blocks:
+          return process_children(node) if parent_type?(Prism::BlockNode)
+
+          # Only wrap super calls that have a block attached
+          # Bare super calls (with or without arguments) are just delegation and shouldn't be instrumented
+          if node.block
+            wrap_node(node)
+          end
+
+          # Process children to handle nested calls, but blocks attached to this super
+          # won't be wrapped separately (handled by visit_block_node check)
+          process_children(node)
+        end
+
+        def visit_forwarding_super_node(node)
+          # We aren't interested in top level super calls:
+          return process_children(node) if @method.empty?
+
+          # This ignores super calls inside blocks:
+          return process_children(node) if parent_type?(Prism::BlockNode)
+
+          # Only wrap super calls that have a block attached
+          # Bare super calls are just delegation and shouldn't be instrumented
+          if node.block
+            wrap_node(node)
+          end
+
+          # Process children to handle nested calls, but blocks attached to this super
+          # won't be wrapped separately (handled by visit_block_node check)
+          process_children(node)
+        end
+
+        # This is meant to mirror that of the parser implementation.
+        # See test/unit/auto_instrument/hash_shorthand_controller-instrumented.rb
+        # Non-nil receiver is handled in visit_call_node.
+        def visit_hash_node(node)
+          # If this hash is a descendant of a CallNode (at any level), don't instrument individual elements
+          # The parent CallNode will wrap the entire expression
+          # This allows hashes in local variable assignments to be instrumented,
+          # but hashes in method calls to be wrapped as a unit
+          in_call_node = @nesting.any? { |n| n.is_a?(Prism::CallNode) }
+
+          node.elements.each do |element|
+            if element.is_a?(Prism::AssocNode) && element.key.is_a?(Prism::SymbolNode)
+              value_node = element.value
+
+              # Only instrument hash element values if we're not in a CallNode
+              # Handles shorthand syntax like `shorthand:` → line 6
+              if !in_call_node && value_node.is_a?(Prism::ImplicitNode)
+                key = element.key.unescaped
+                inner_call = value_node.value
+
+                line = element.location.start_line
+                source = @code[element.location.start_offset...element.location.end_offset]
+                file_name = @path
+                method_name = @method.last.name
+                bt = ["#{file_name}:#{line}:in `#{method_name}'"]
+
+                instrument_before = "::ScoutApm::AutoInstrument(#{source.dump},#{bt}){"
+                instrument_after = "}"
+                new_text = "#{key}: #{instrument_before}#{key}#{instrument_after}"
+                add_replacement(element.location.start_offset, element.location.end_offset, new_text)
+
+                @instrumented_nodes.add(value_node)
+                @instrumented_nodes.add(inner_call)
+                next
+              elsif !in_call_node && value_node.is_a?(Prism::CallNode) && value_node.receiver.nil?
+                line = element.location.start_line
+                key = element.key.unescaped
+                value_name = value_node.name.to_s
+                pair_source = @code[element.location.start_offset...element.location.end_offset]
+                value_source = @code[value_node.location.start_offset...value_node.location.end_offset]
+                key_source = @code[element.key.location.start_offset...element.key.location.end_offset]
+                file_name = @path
+                method_name = @method.last.name
+                bt = ["#{file_name}:#{line}:in `#{method_name}'"]
+
+                has_arguments = value_node.arguments && !value_node.arguments.arguments.empty?
+
+                # Handles hash_rocket w/ same key/value name and no arguments.
+                # See test for more info on backward compatibility on this one.
+                # e.g. `hash_rocket: hash_rocket` → line 9
+                if key == value_name && !has_arguments && key_source.start_with?(':')
+                  source_for_dump = pair_source
+                  instrument_before = "::ScoutApm::AutoInstrument(#{source_for_dump.dump},#{bt}){"
+                  instrument_after = "}"
+                  instrumented_value = "#{instrument_before}#{value_source}#{instrument_after}"
+                  new_text = "#{key}: #{instrumented_value}"
+                  add_replacement(element.location.start_offset, element.location.end_offset, new_text)
+
+                # If key == value_name and no arguments → direct shorthand pair
+                # e.g. `longhand: longhand` → line 7
+                elsif key == value_name && !has_arguments && !key_source.start_with?(':')
+                  source_for_dump = pair_source
+                  instrument_before = "::ScoutApm::AutoInstrument(#{source_for_dump.dump},#{bt}){"
+                  instrument_after = "}"
+                  instrumented_value = "#{instrument_before}#{value_source}#{instrument_after}"
+                  add_replacement(value_node.location.start_offset, value_node.location.end_offset, instrumented_value)
+
+                # If key != value_name → “different key/value name” case
+                # e.g. `longhand_different_key: longhand` → line 8  
+                # or `:hash_rocket_different_key => hash_rocket` → line 10
+                elsif key != value_name && !has_arguments
+                  source_for_dump = value_source
+                  instrument_before = "::ScoutApm::AutoInstrument(#{source_for_dump.dump},#{bt}){"
+                  instrument_after = "}"
+                  instrumented_value = "#{instrument_before}#{value_source}#{instrument_after}"
+                  add_replacement(value_node.location.start_offset, value_node.location.end_offset, instrumented_value)
+
+                # If value_node has arguments → method call with params
+                # e.g. `nested_call(params["timestamp"])` → line 15
+                elsif has_arguments
+                  source_for_dump = value_source
+                  instrument_before = "::ScoutApm::AutoInstrument(#{source_for_dump.dump},#{bt}){"
+                  instrument_after = "}"
+                  instrumented_value = "#{instrument_before}#{value_source}#{instrument_after}"
+                  add_replacement(value_node.location.start_offset, value_node.location.end_offset, instrumented_value)
+                end
+
+                @instrumented_nodes.add(value_node)
+                next
+              end
+            end
+
+            element.compact_child_nodes.each do |child|
+              process(child)
+            end
+          end
+        end
+
+        def process(node)
+          return unless node
+
+          # We are nesting inside this node:
+          @nesting.push(node)
+
+          case node
+          when Prism::DefNode
+            # If the node is a method, push it on the method stack as well:
+            @method.push(node)
+            process_children(node)
+            @method.pop
+          when Prism::ClassNode
+            # If the node is a method, push it on the scope stack as well:
+            @scope.push(node.name)
+            process_children(node)
+            @scope.pop
+          when Prism::BlockNode
+            visit_block_node(node)
+          when Prism::MultiTargetNode
+            visit_multi_target_node(node)
+          when Prism::CallNode
+            visit_call_node(node)
+          when Prism::SuperNode
+            visit_super_node(node)
+          when Prism::ForwardingSuperNode
+            visit_forwarding_super_node(node)
+          when Prism::HashNode
+            visit_hash_node(node)
+          when Prism::CallOperatorWriteNode, Prism::CallOrWriteNode, Prism::CallAndWriteNode
+            # For op assignment nodes, only process the value
+            process(node.value)
+          else
+            process_children(node)
+          end
+
+          @nesting.pop
+        end
+
+        def process_children(node)
+          node.compact_child_nodes.each do |child|
+            process(child)
+          end
+        end
+      end
+    end
+
+    class Cache
+      def initialize
+        @local_assignments = {}
+      end
+
+      def local_assignments?(node)
+        return false unless node
+        return false unless node.respond_to?(:compact_child_nodes)
+
+        unless @local_assignments.key?(node)
+          if node.is_a?(Prism::LocalVariableWriteNode)
+            @local_assignments[node] = true
+          elsif node.compact_child_nodes.find{|child|
+            # Don't check blocks - assignments inside blocks shouldn't affect the parent call
+            next if child.is_a?(Prism::BlockNode)
+            self.local_assignments?(child)
+          }
+            @local_assignments[node] = true
+          else
+            @local_assignments[node] = false
+          end
+        end
+
+        return @local_assignments[node]
+      end
+    end
+  end
+end